Note: This is test shard 8 of 8.
[==========] Running 7 tests from 6 test suites.
[----------] Global test environment set-up.
[----------] 1 test from TableRangeRebalancingTest
[ RUN ] TableRangeRebalancingTest.Basic
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240502 14:14:15.162024 28530 test_util.cc:268] Using random seed: 936416380
I20240502 14:14:15.173175 28530 ts_itest-base.cc:112] Starting cluster with:
I20240502 14:14:15.173257 28530 ts_itest-base.cc:113] --------------
I20240502 14:14:15.173302 28530 ts_itest-base.cc:114] 3 tablet servers
I20240502 14:14:15.173367 28530 ts_itest-base.cc:115] 3 replicas per TS
I20240502 14:14:15.173450 28530 ts_itest-base.cc:116] --------------
2024-05-02T14:14:15Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T14:14:15Z Disabled control of system clock
I20240502 14:14:15.198266 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:41663
--webserver_interface=127.27.220.190
--webserver_port=0
--builtin_ntp_servers=127.27.220.148:35383
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.27.220.190:41663
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 14:14:15.515949 28542 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:15.516274 28542 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:15.516371 28542 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:15.523627 28542 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:15.523759 28542 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 14:14:15.523810 28542 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:15.523859 28542 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 14:14:15.523900 28542 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 14:14:15.532858 28542 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:35383
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.27.220.190:41663
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:41663
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.27.220.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:15.534232 28542 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:15.535995 28542 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:15.547024 28548 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:15.547657 28547 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:15.548132 28550 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:15.548956 28542 server_base.cc:1034] running on GCE node
I20240502 14:14:15.549800 28542 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:15.551337 28542 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:15.552563 28542 hybrid_clock.cc:648] HybridClock initialized: now 1714659255552515 us; error 50 us; skew 500 ppm
I20240502 14:14:15.552973 28542 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:15.559068 28542 webserver.cc:460] Webserver started at http://127.27.220.190:41393/ using document root <none> and password file <none>
I20240502 14:14:15.559862 28542 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:15.559993 28542 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:15.560322 28542 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:15.563253 28542 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "a3a9e45102a9428d9df60f76ee10fa7f"
format_stamp: "Formatted at 2024-05-02 14:14:15 on dist-test-slave-j72s"
I20240502 14:14:15.564014 28542 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "a3a9e45102a9428d9df60f76ee10fa7f"
format_stamp: "Formatted at 2024-05-02 14:14:15 on dist-test-slave-j72s"
I20240502 14:14:15.569347 28542 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.001s sys 0.006s
I20240502 14:14:15.572811 28556 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:15.574215 28542 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:15.574373 28542 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
uuid: "a3a9e45102a9428d9df60f76ee10fa7f"
format_stamp: "Formatted at 2024-05-02 14:14:15 on dist-test-slave-j72s"
I20240502 14:14:15.574559 28542 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:15.599647 28542 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:15.600497 28542 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:15.600785 28542 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:15.618701 28542 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.190:41663
I20240502 14:14:15.618703 28607 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.190:41663 every 8 connection(s)
I20240502 14:14:15.619938 28542 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 14:14:15.623427 28608 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:15.624641 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 28542
I20240502 14:14:15.624884 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 14:14:15.633911 28608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f: Bootstrap starting.
I20240502 14:14:15.636732 28608 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:15.637729 28608 log.cc:824] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:15.640290 28608 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f: No bootstrap required, opened a new log
I20240502 14:14:15.646358 28608 raft_consensus.cc:348] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a3a9e45102a9428d9df60f76ee10fa7f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 41663 } }
I20240502 14:14:15.646698 28608 raft_consensus.cc:374] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:15.646806 28608 raft_consensus.cc:729] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3a9e45102a9428d9df60f76ee10fa7f, State: Initialized, Role: FOLLOWER
I20240502 14:14:15.647450 28608 consensus_queue.cc:260] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a3a9e45102a9428d9df60f76ee10fa7f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 41663 } }
I20240502 14:14:15.647687 28608 raft_consensus.cc:388] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:14:15.647819 28608 raft_consensus.cc:482] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:14:15.647974 28608 raft_consensus.cc:3037] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:15.650796 28608 raft_consensus.cc:504] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a3a9e45102a9428d9df60f76ee10fa7f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 41663 } }
I20240502 14:14:15.651259 28608 leader_election.cc:302] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: a3a9e45102a9428d9df60f76ee10fa7f; no voters:
I20240502 14:14:15.651959 28608 leader_election.cc:288] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:14:15.652210 28613 raft_consensus.cc:2781] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:15.653070 28613 raft_consensus.cc:686] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [term 1 LEADER]: Becoming Leader. State: Replica: a3a9e45102a9428d9df60f76ee10fa7f, State: Running, Role: LEADER
I20240502 14:14:15.653617 28613 consensus_queue.cc:237] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a3a9e45102a9428d9df60f76ee10fa7f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 41663 } }
I20240502 14:14:15.653962 28608 sys_catalog.cc:564] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [sys.catalog]: configured and running, proceeding with master startup.
I20240502 14:14:15.656306 28614 sys_catalog.cc:455] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "a3a9e45102a9428d9df60f76ee10fa7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a3a9e45102a9428d9df60f76ee10fa7f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 41663 } } }
I20240502 14:14:15.656668 28614 sys_catalog.cc:458] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [sys.catalog]: This master's current role is: LEADER
I20240502 14:14:15.656312 28615 sys_catalog.cc:455] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [sys.catalog]: SysCatalogTable state changed. Reason: New leader a3a9e45102a9428d9df60f76ee10fa7f. Latest consensus state: current_term: 1 leader_uuid: "a3a9e45102a9428d9df60f76ee10fa7f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "a3a9e45102a9428d9df60f76ee10fa7f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 41663 } } }
I20240502 14:14:15.662432 28615 sys_catalog.cc:458] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f [sys.catalog]: This master's current role is: LEADER
W20240502 14:14:15.666857 28629 catalog_manager.cc:1558] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20240502 14:14:15.666992 28629 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20240502 14:14:15.667136 28619 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 14:14:15.668680 28619 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 14:14:15.676409 28619 catalog_manager.cc:1347] Generated new cluster ID: cc46e11f17fe426ca7a72b89ab0fc663
I20240502 14:14:15.676535 28619 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 14:14:15.686573 28619 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 14:14:15.687809 28619 catalog_manager.cc:1504] Loading token signing keys...
I20240502 14:14:15.698457 28619 catalog_manager.cc:5857] T 00000000000000000000000000000000 P a3a9e45102a9428d9df60f76ee10fa7f: Generated new TSK 0
I20240502 14:14:15.699299 28619 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 14:14:15.704926 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.129:0
--local_ip_for_outbound_sockets=127.27.220.129
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:41663
--builtin_ntp_servers=127.27.220.148:35383
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:14:16.027948 28632 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:16.028240 28632 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:16.028334 28632 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:16.035642 28632 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:16.035761 28632 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:16.035913 28632 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.129
I20240502 14:14:16.045068 28632 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:35383
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:41663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.129
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:16.046437 28632 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:16.048152 28632 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:16.060559 28637 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:16.060962 28638 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:16.061638 28640 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:16.062165 28632 server_base.cc:1034] running on GCE node
I20240502 14:14:16.062726 28632 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:16.063622 28632 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:16.064813 28632 hybrid_clock.cc:648] HybridClock initialized: now 1714659256064755 us; error 49 us; skew 500 ppm
I20240502 14:14:16.065114 28632 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:16.067332 28632 webserver.cc:460] Webserver started at http://127.27.220.129:38233/ using document root <none> and password file <none>
I20240502 14:14:16.068048 28632 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:16.068161 28632 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:16.068466 28632 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:16.071079 28632 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "c9eb1da45f824ba89668a67f863f6c03"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.071791 28632 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "c9eb1da45f824ba89668a67f863f6c03"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.076922 28632 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240502 14:14:16.080569 28646 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.081841 28632 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:16.082002 28632 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "c9eb1da45f824ba89668a67f863f6c03"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.082180 28632 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:16.094075 28632 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:16.094746 28632 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:16.094991 28632 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:16.095773 28632 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:16.097133 28632 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:16.097237 28632 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.097347 28632 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:16.097419 28632 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.137246 28632 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.129:33477
I20240502 14:14:16.137300 28758 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.129:33477 every 8 connection(s)
I20240502 14:14:16.138521 28632 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 14:14:16.143656 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 28632
I20240502 14:14:16.143886 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 14:14:16.147389 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.130:0
--local_ip_for_outbound_sockets=127.27.220.130
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:41663
--builtin_ntp_servers=127.27.220.148:35383
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:16.151803 28759 heartbeater.cc:347] Connected to a master server at 127.27.220.190:41663
I20240502 14:14:16.152133 28759 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:16.152920 28759 heartbeater.cc:511] Master 127.27.220.190:41663 requested a full tablet report, sending...
I20240502 14:14:16.155134 28573 ts_manager.cc:194] Registered new tserver with Master: c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477)
I20240502 14:14:16.157303 28573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.129:38637
W20240502 14:14:16.459199 28762 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:16.459482 28762 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:16.459568 28762 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:16.466833 28762 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:16.466960 28762 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:16.467078 28762 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.130
I20240502 14:14:16.478384 28762 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:35383
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:41663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.130
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:16.479853 28762 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:16.481627 28762 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:16.493788 28767 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:16.494132 28768 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:16.494643 28770 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:16.495306 28762 server_base.cc:1034] running on GCE node
I20240502 14:14:16.495963 28762 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:16.496943 28762 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:16.498170 28762 hybrid_clock.cc:648] HybridClock initialized: now 1714659256498108 us; error 50 us; skew 500 ppm
I20240502 14:14:16.498476 28762 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:16.500612 28762 webserver.cc:460] Webserver started at http://127.27.220.130:40083/ using document root <none> and password file <none>
I20240502 14:14:16.501538 28762 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:16.501667 28762 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:16.502027 28762 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:16.504462 28762 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "d3ff4b86b3e846c9ba04b989f823a000"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.505198 28762 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "d3ff4b86b3e846c9ba04b989f823a000"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.510211 28762 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.003s
I20240502 14:14:16.513439 28776 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.514690 28762 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 14:14:16.514842 28762 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "d3ff4b86b3e846c9ba04b989f823a000"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.515022 28762 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:16.526767 28762 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:16.527519 28762 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:16.527784 28762 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:16.528628 28762 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:16.530000 28762 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:16.530093 28762 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.530198 28762 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:16.530264 28762 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.560204 28762 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.130:41255
I20240502 14:14:16.560245 28888 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.130:41255 every 8 connection(s)
I20240502 14:14:16.561447 28762 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 14:14:16.566546 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 28762
I20240502 14:14:16.566709 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 14:14:16.569927 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:0
--local_ip_for_outbound_sockets=127.27.220.131
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:41663
--builtin_ntp_servers=127.27.220.148:35383
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:16.573275 28889 heartbeater.cc:347] Connected to a master server at 127.27.220.190:41663
I20240502 14:14:16.573591 28889 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:16.574398 28889 heartbeater.cc:511] Master 127.27.220.190:41663 requested a full tablet report, sending...
I20240502 14:14:16.575718 28573 ts_manager.cc:194] Registered new tserver with Master: d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:16.576848 28573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.130:42395
W20240502 14:14:16.874213 28892 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:16.874470 28892 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:16.874548 28892 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:16.881651 28892 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:16.881793 28892 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:16.881903 28892 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:14:16.890822 28892 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:35383
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:41663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:16.892282 28892 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:16.894013 28892 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:16.905938 28900 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:16.906436 28898 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:16.906903 28897 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:16.907583 28892 server_base.cc:1034] running on GCE node
I20240502 14:14:16.908107 28892 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:16.908983 28892 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:16.910168 28892 hybrid_clock.cc:648] HybridClock initialized: now 1714659256910126 us; error 33 us; skew 500 ppm
I20240502 14:14:16.910485 28892 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:16.912591 28892 webserver.cc:460] Webserver started at http://127.27.220.131:37141/ using document root <none> and password file <none>
I20240502 14:14:16.913290 28892 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:16.913393 28892 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:16.913754 28892 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:16.916265 28892 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "f14e011f333f4dbeb909c21e11b48ea4"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.916980 28892 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "f14e011f333f4dbeb909c21e11b48ea4"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.921990 28892 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240502 14:14:16.925189 28906 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.926524 28892 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:16.926671 28892 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "f14e011f333f4dbeb909c21e11b48ea4"
format_stamp: "Formatted at 2024-05-02 14:14:16 on dist-test-slave-j72s"
I20240502 14:14:16.926858 28892 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:16.961632 28892 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:16.962520 28892 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:16.962805 28892 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:16.963812 28892 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:16.965253 28892 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:16.965339 28892 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.965456 28892 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:16.965550 28892 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:16.997786 28892 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:38563
I20240502 14:14:16.997833 29018 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:38563 every 8 connection(s)
I20240502 14:14:16.999081 28892 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:14:17.008394 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 28892
I20240502 14:14:17.008594 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 14:14:17.011431 29019 heartbeater.cc:347] Connected to a master server at 127.27.220.190:41663
I20240502 14:14:17.011703 29019 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:17.012506 29019 heartbeater.cc:511] Master 127.27.220.190:41663 requested a full tablet report, sending...
I20240502 14:14:17.013850 28573 ts_manager.cc:194] Registered new tserver with Master: f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:17.014497 28573 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:43519
I20240502 14:14:17.024636 28530 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
I20240502 14:14:17.044091 28530 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
I20240502 14:14:17.160421 28759 heartbeater.cc:503] Master 127.27.220.190:41663 was elected leader, sending a full tablet report...
I20240502 14:14:17.579676 28889 heartbeater.cc:503] Master 127.27.220.190:41663 was elected leader, sending a full tablet report...
I20240502 14:14:18.016706 29019 heartbeater.cc:503] Master 127.27.220.190:41663 was elected leader, sending a full tablet report...
I20240502 14:14:20.798792 28573 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33832:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
rows: "<redacted>""\006\001\000\000\000\000\200\007\001\000\000\000\000\000\006\001\000\000\000\000\000\007\001\000\377\377\377\177"
indirect_data: "<redacted>"""
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 8
seed: 0
}
}
W20240502 14:14:20.805466 28573 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240502 14:14:20.841867 28823 tablet_service.cc:1467] Processing CreateTablet for tablet bb281e01d01f457f84364f0b432a676e (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.842597 28693 tablet_service.cc:1467] Processing CreateTablet for tablet bb281e01d01f457f84364f0b432a676e (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.845237 28690 tablet_service.cc:1467] Processing CreateTablet for tablet a1a7393c68b047b2a01e26455d8b27ae (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.845453 28693 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bb281e01d01f457f84364f0b432a676e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.845997 28692 tablet_service.cc:1467] Processing CreateTablet for tablet 8234addac59a4ca299a252c1725e7333 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.846565 28692 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8234addac59a4ca299a252c1725e7333. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.847323 28691 tablet_service.cc:1467] Processing CreateTablet for tablet 222aabe6414f498fa3749f2f83873903 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.847766 28691 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 222aabe6414f498fa3749f2f83873903. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.849527 28688 tablet_service.cc:1467] Processing CreateTablet for tablet 46170e7b56224a418997a98a85422e02 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.849812 28690 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1a7393c68b047b2a01e26455d8b27ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.850172 28688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 46170e7b56224a418997a98a85422e02. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.850404 28694 tablet_service.cc:1467] Processing CreateTablet for tablet 3e29f784041244d6906d011a85b5725a (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.850697 28689 tablet_service.cc:1467] Processing CreateTablet for tablet 207e9900d5d34ccda396e881a0134ea2 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.850998 28694 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e29f784041244d6906d011a85b5725a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.851241 28689 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 207e9900d5d34ccda396e881a0134ea2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.852463 28684 tablet_service.cc:1467] Processing CreateTablet for tablet f7a111dfadd448bab1d5ffdffd2a9fbc (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.852572 28685 tablet_service.cc:1467] Processing CreateTablet for tablet 2bcfe7bce7af41b79ed1415e7a8b76ed (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.853003 28684 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f7a111dfadd448bab1d5ffdffd2a9fbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.853502 28683 tablet_service.cc:1467] Processing CreateTablet for tablet f2b2fd9b7b3c43928513bfe289b87e80 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.853996 28683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2b2fd9b7b3c43928513bfe289b87e80. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.854866 28681 tablet_service.cc:1467] Processing CreateTablet for tablet de95686c7daa435f9627f8dca3fdaa18 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.855043 28682 tablet_service.cc:1467] Processing CreateTablet for tablet ab370a1503e141c0a807d33964687fc9 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.855448 28681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet de95686c7daa435f9627f8dca3fdaa18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.855468 28679 tablet_service.cc:1467] Processing CreateTablet for tablet bc33ffa96d0d4b498f44531cb67c342c (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.855674 28687 tablet_service.cc:1467] Processing CreateTablet for tablet fec9c54b0b9041eeab5a4a45dbdf1a6e (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.843505 28822 tablet_service.cc:1467] Processing CreateTablet for tablet 8234addac59a4ca299a252c1725e7333 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.856014 28685 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2bcfe7bce7af41b79ed1415e7a8b76ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.856168 28680 tablet_service.cc:1467] Processing CreateTablet for tablet 840e603053f94d7192f26b0dfda184f6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.844617 28824 tablet_service.cc:1467] Processing CreateTablet for tablet 3e29f784041244d6906d011a85b5725a (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.851245 28686 tablet_service.cc:1467] Processing CreateTablet for tablet 298bfef2fe6c4a9593e3f795ae6753a0 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.856552 28809 tablet_service.cc:1467] Processing CreateTablet for tablet bc33ffa96d0d4b498f44531cb67c342c (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.856958 28682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ab370a1503e141c0a807d33964687fc9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.857110 28822 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8234addac59a4ca299a252c1725e7333. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.857232 28679 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc33ffa96d0d4b498f44531cb67c342c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.857560 28687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fec9c54b0b9041eeab5a4a45dbdf1a6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.857685 28824 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e29f784041244d6906d011a85b5725a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.857928 28809 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc33ffa96d0d4b498f44531cb67c342c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.858177 28680 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 840e603053f94d7192f26b0dfda184f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.847651 28821 tablet_service.cc:1467] Processing CreateTablet for tablet 222aabe6414f498fa3749f2f83873903 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.858911 28686 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 298bfef2fe6c4a9593e3f795ae6753a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.858955 28821 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 222aabe6414f498fa3749f2f83873903. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.848258 28820 tablet_service.cc:1467] Processing CreateTablet for tablet a1a7393c68b047b2a01e26455d8b27ae (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.848800 28819 tablet_service.cc:1467] Processing CreateTablet for tablet 207e9900d5d34ccda396e881a0134ea2 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.863901 28819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 207e9900d5d34ccda396e881a0134ea2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.849377 28818 tablet_service.cc:1467] Processing CreateTablet for tablet 46170e7b56224a418997a98a85422e02 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.869243 28818 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 46170e7b56224a418997a98a85422e02. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.849967 28817 tablet_service.cc:1467] Processing CreateTablet for tablet fec9c54b0b9041eeab5a4a45dbdf1a6e (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.869959 28817 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fec9c54b0b9041eeab5a4a45dbdf1a6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.850545 28816 tablet_service.cc:1467] Processing CreateTablet for tablet 298bfef2fe6c4a9593e3f795ae6753a0 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.873778 28816 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 298bfef2fe6c4a9593e3f795ae6753a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.851116 28815 tablet_service.cc:1467] Processing CreateTablet for tablet 2bcfe7bce7af41b79ed1415e7a8b76ed (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.875402 28815 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2bcfe7bce7af41b79ed1415e7a8b76ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.851702 28814 tablet_service.cc:1467] Processing CreateTablet for tablet f7a111dfadd448bab1d5ffdffd2a9fbc (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.852285 28813 tablet_service.cc:1467] Processing CreateTablet for tablet f2b2fd9b7b3c43928513bfe289b87e80 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.852887 28812 tablet_service.cc:1467] Processing CreateTablet for tablet ab370a1503e141c0a807d33964687fc9 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.854339 28811 tablet_service.cc:1467] Processing CreateTablet for tablet de95686c7daa435f9627f8dca3fdaa18 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.855587 28810 tablet_service.cc:1467] Processing CreateTablet for tablet 840e603053f94d7192f26b0dfda184f6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.878027 28820 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1a7393c68b047b2a01e26455d8b27ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.881830 28814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f7a111dfadd448bab1d5ffdffd2a9fbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.882404 28812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ab370a1503e141c0a807d33964687fc9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.882930 28813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2b2fd9b7b3c43928513bfe289b87e80. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.883437 28811 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet de95686c7daa435f9627f8dca3fdaa18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.883952 28810 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 840e603053f94d7192f26b0dfda184f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.860708 28952 tablet_service.cc:1467] Processing CreateTablet for tablet 8234addac59a4ca299a252c1725e7333 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.865559 28942 tablet_service.cc:1467] Processing CreateTablet for tablet ab370a1503e141c0a807d33964687fc9 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.865803 28940 tablet_service.cc:1467] Processing CreateTablet for tablet 840e603053f94d7192f26b0dfda184f6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.865299 28945 tablet_service.cc:1467] Processing CreateTablet for tablet 2bcfe7bce7af41b79ed1415e7a8b76ed (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.865408 28939 tablet_service.cc:1467] Processing CreateTablet for tablet bc33ffa96d0d4b498f44531cb67c342c (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.865180 28944 tablet_service.cc:1467] Processing CreateTablet for tablet f7a111dfadd448bab1d5ffdffd2a9fbc (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.865932 28943 tablet_service.cc:1467] Processing CreateTablet for tablet f2b2fd9b7b3c43928513bfe289b87e80 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.864956 28941 tablet_service.cc:1467] Processing CreateTablet for tablet de95686c7daa435f9627f8dca3fdaa18 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.865667 28947 tablet_service.cc:1467] Processing CreateTablet for tablet fec9c54b0b9041eeab5a4a45dbdf1a6e (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.896944 28952 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8234addac59a4ca299a252c1725e7333. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.864753 28946 tablet_service.cc:1467] Processing CreateTablet for tablet 298bfef2fe6c4a9593e3f795ae6753a0 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.865058 28948 tablet_service.cc:1467] Processing CreateTablet for tablet 46170e7b56224a418997a98a85422e02 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.862113 28949 tablet_service.cc:1467] Processing CreateTablet for tablet 207e9900d5d34ccda396e881a0134ea2 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.897979 28939 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bc33ffa96d0d4b498f44531cb67c342c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.894933 28953 tablet_service.cc:1467] Processing CreateTablet for tablet bb281e01d01f457f84364f0b432a676e (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.899363 28953 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bb281e01d01f457f84364f0b432a676e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.861410 28954 tablet_service.cc:1467] Processing CreateTablet for tablet 3e29f784041244d6906d011a85b5725a (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.900282 28954 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e29f784041244d6906d011a85b5725a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.901821 28944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f7a111dfadd448bab1d5ffdffd2a9fbc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.902567 28943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f2b2fd9b7b3c43928513bfe289b87e80. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.903721 28941 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet de95686c7daa435f9627f8dca3fdaa18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.904515 28947 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fec9c54b0b9041eeab5a4a45dbdf1a6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.905270 28946 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 298bfef2fe6c4a9593e3f795ae6753a0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.906092 28948 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 46170e7b56224a418997a98a85422e02. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.907032 28940 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 840e603053f94d7192f26b0dfda184f6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.907791 28949 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 207e9900d5d34ccda396e881a0134ea2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.908629 28942 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ab370a1503e141c0a807d33964687fc9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.909373 28945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2bcfe7bce7af41b79ed1415e7a8b76ed. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.861764 28950 tablet_service.cc:1467] Processing CreateTablet for tablet a1a7393c68b047b2a01e26455d8b27ae (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 14:14:20.910375 28950 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1a7393c68b047b2a01e26455d8b27ae. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.882697 28951 tablet_service.cc:1467] Processing CreateTablet for tablet 222aabe6414f498fa3749f2f83873903 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0ac9c0acd4734ea5a472dacc2031968b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 14:14:20.911332 28951 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 222aabe6414f498fa3749f2f83873903. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.926251 29038 tablet_bootstrap.cc:492] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:20.934904 28823 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bb281e01d01f457f84364f0b432a676e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:20.948959 29038 tablet_bootstrap.cc:654] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:20.950771 29038 log.cc:824] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:20.966208 29039 tablet_bootstrap.cc:492] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:20.971117 29039 tablet_bootstrap.cc:654] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:20.972071 29037 tablet_bootstrap.cc:492] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:20.974081 29039 log.cc:824] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:20.975800 29037 tablet_bootstrap.cc:654] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:20.977428 29037 log.cc:824] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:20.988379 29037 tablet_bootstrap.cc:492] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:20.988847 29037 ts_tablet_manager.cc:1367] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.017s user 0.010s sys 0.005s
I20240502 14:14:20.998199 29037 raft_consensus.cc:348] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:20.998924 29037 raft_consensus.cc:374] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:20.999092 29037 raft_consensus.cc:729] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.000128 29037 consensus_queue.cc:260] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.009207 29037 ts_tablet_manager.cc:1398] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.020s user 0.011s sys 0.006s
I20240502 14:14:21.009768 29037 tablet_bootstrap.cc:492] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.011566 29037 tablet_bootstrap.cc:654] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.019884 29037 tablet_bootstrap.cc:492] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.020114 29037 ts_tablet_manager.cc:1367] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.008s
I20240502 14:14:21.021216 29037 raft_consensus.cc:348] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.021493 29037 raft_consensus.cc:374] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.021659 29037 raft_consensus.cc:729] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.021981 29037 consensus_queue.cc:260] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.023840 29037 ts_tablet_manager.cc:1398] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 14:14:21.024161 29037 tablet_bootstrap.cc:492] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.026252 29037 tablet_bootstrap.cc:654] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.033859 29037 tablet_bootstrap.cc:492] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.034039 29037 ts_tablet_manager.cc:1367] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.005s
I20240502 14:14:21.034700 29037 raft_consensus.cc:348] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.035007 29037 raft_consensus.cc:374] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.035043 29039 tablet_bootstrap.cc:492] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.035110 29037 raft_consensus.cc:729] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.035423 29039 ts_tablet_manager.cc:1367] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.070s user 0.017s sys 0.049s
I20240502 14:14:21.035389 29037 consensus_queue.cc:260] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.035908 29038 tablet_bootstrap.cc:492] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.035919 29037 ts_tablet_manager.cc:1398] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.036228 29037 tablet_bootstrap.cc:492] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.036239 29038 ts_tablet_manager.cc:1367] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.111s user 0.039s sys 0.037s
I20240502 14:14:21.037943 29037 tablet_bootstrap.cc:654] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.043944 29037 tablet_bootstrap.cc:492] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.044097 29037 ts_tablet_manager.cc:1367] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240502 14:14:21.044667 29037 raft_consensus.cc:348] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.044840 29037 raft_consensus.cc:374] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.044605 29039 raft_consensus.cc:348] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.044972 29037 raft_consensus.cc:729] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.045078 29039 raft_consensus.cc:374] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.045219 29039 raft_consensus.cc:729] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.045213 29037 consensus_queue.cc:260] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.045204 29038 raft_consensus.cc:348] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.045739 29038 raft_consensus.cc:374] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.045812 29037 ts_tablet_manager.cc:1398] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240502 14:14:21.045892 29038 raft_consensus.cc:729] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.045997 29039 consensus_queue.cc:260] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.046154 29037 tablet_bootstrap.cc:492] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.046878 29038 consensus_queue.cc:260] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.047870 29037 tablet_bootstrap.cc:654] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.052150 29039 ts_tablet_manager.cc:1398] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.017s user 0.009s sys 0.007s
I20240502 14:14:21.052592 29039 tablet_bootstrap.cc:492] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.052897 29037 tablet_bootstrap.cc:492] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.053053 29037 ts_tablet_manager.cc:1367] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.006s
I20240502 14:14:21.053856 29037 raft_consensus.cc:348] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.054143 29037 raft_consensus.cc:374] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.054306 29037 raft_consensus.cc:729] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.054376 29039 tablet_bootstrap.cc:654] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.054700 29037 consensus_queue.cc:260] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.054953 29038 ts_tablet_manager.cc:1398] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.019s user 0.015s sys 0.000s
I20240502 14:14:21.055450 29038 tablet_bootstrap.cc:492] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.056253 29037 ts_tablet_manager.cc:1398] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:21.056586 29037 tablet_bootstrap.cc:492] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.058252 29037 tablet_bootstrap.cc:654] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.058472 29038 tablet_bootstrap.cc:654] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.065899 29039 tablet_bootstrap.cc:492] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.066071 29039 ts_tablet_manager.cc:1367] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.014s user 0.008s sys 0.004s
I20240502 14:14:21.066113 29038 tablet_bootstrap.cc:492] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.066257 29038 ts_tablet_manager.cc:1367] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.011s user 0.007s sys 0.001s
W20240502 14:14:21.066232 28890 tablet.cc:2369] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:14:21.067068 29037 tablet_bootstrap.cc:492] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.067219 29037 ts_tablet_manager.cc:1367] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.004s
I20240502 14:14:21.067147 29039 raft_consensus.cc:348] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.067170 29038 raft_consensus.cc:348] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.067377 29038 raft_consensus.cc:374] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.067426 29039 raft_consensus.cc:374] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.067495 29038 raft_consensus.cc:729] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.067556 29039 raft_consensus.cc:729] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.067785 29039 consensus_queue.cc:260] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.067785 29038 consensus_queue.cc:260] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.068030 29037 raft_consensus.cc:348] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.068267 29037 raft_consensus.cc:374] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.068382 29037 raft_consensus.cc:729] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.068420 29039 ts_tablet_manager.cc:1398] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.068426 29038 ts_tablet_manager.cc:1398] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:14:21.068766 29038 tablet_bootstrap.cc:492] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.068630 29037 consensus_queue.cc:260] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.069307 29037 ts_tablet_manager.cc:1398] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:14:21.069628 29037 tablet_bootstrap.cc:492] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.071282 29039 tablet_bootstrap.cc:492] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.071517 29037 tablet_bootstrap.cc:654] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.071624 29038 tablet_bootstrap.cc:654] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.073899 29043 raft_consensus.cc:482] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.074175 29043 raft_consensus.cc:504] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.074440 29039 tablet_bootstrap.cc:654] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.076577 29043 leader_election.cc:288] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.077805 29037 tablet_bootstrap.cc:492] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.078075 29037 ts_tablet_manager.cc:1367] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.004s
I20240502 14:14:21.079191 29037 raft_consensus.cc:348] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.079605 29037 raft_consensus.cc:374] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.079746 29037 raft_consensus.cc:729] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.080200 29037 consensus_queue.cc:260] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.080940 29037 ts_tablet_manager.cc:1398] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:14:21.081300 29037 tablet_bootstrap.cc:492] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.083860 29037 tablet_bootstrap.cc:654] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.085100 29038 tablet_bootstrap.cc:492] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.085343 29038 ts_tablet_manager.cc:1367] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.017s user 0.003s sys 0.001s
I20240502 14:14:21.088117 29038 raft_consensus.cc:348] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.093127 29037 tablet_bootstrap.cc:492] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.093215 29038 raft_consensus.cc:374] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.093293 29037 ts_tablet_manager.cc:1367] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
I20240502 14:14:21.093324 29038 raft_consensus.cc:729] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.093629 29038 consensus_queue.cc:260] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.093808 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.094239 29038 ts_tablet_manager.cc:1398] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.009s user 0.000s sys 0.004s
I20240502 14:14:21.094404 28974 raft_consensus.cc:2454] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.094533 29038 tablet_bootstrap.cc:492] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.094676 29037 raft_consensus.cc:348] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.094883 29037 raft_consensus.cc:374] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.094975 29037 raft_consensus.cc:729] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.095257 29037 consensus_queue.cc:260] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.095844 29037 ts_tablet_manager.cc:1398] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:14:21.096143 29037 tablet_bootstrap.cc:492] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.096262 29038 tablet_bootstrap.cc:654] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.096572 28649 leader_election.cc:302] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.097095 29043 raft_consensus.cc:2781] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.097259 29043 raft_consensus.cc:482] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.097389 29043 raft_consensus.cc:3037] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.099625 29037 tablet_bootstrap.cc:654] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.100203 29039 tablet_bootstrap.cc:492] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.100391 29039 ts_tablet_manager.cc:1367] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.029s user 0.012s sys 0.007s
I20240502 14:14:21.101395 29039 raft_consensus.cc:348] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.101750 29039 raft_consensus.cc:374] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.101856 29039 raft_consensus.cc:729] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.102169 29039 consensus_queue.cc:260] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.102793 29039 ts_tablet_manager.cc:1398] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.103106 29039 tablet_bootstrap.cc:492] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.105347 29039 tablet_bootstrap.cc:654] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.108106 29037 tablet_bootstrap.cc:492] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.108141 29038 tablet_bootstrap.cc:492] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.107872 28844 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.108351 29037 ts_tablet_manager.cc:1367] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.007s
I20240502 14:14:21.108356 29038 ts_tablet_manager.cc:1367] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.001s
I20240502 14:14:21.109059 29037 raft_consensus.cc:348] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.109346 29037 raft_consensus.cc:374] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.109460 29037 raft_consensus.cc:729] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.109444 29038 raft_consensus.cc:348] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.109676 29038 raft_consensus.cc:374] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.109802 29038 raft_consensus.cc:729] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.109757 29037 consensus_queue.cc:260] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
W20240502 14:14:21.109820 28648 leader_election.cc:341] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.110033 29038 consensus_queue.cc:260] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.110913 29038 ts_tablet_manager.cc:1398] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.111234 29038 tablet_bootstrap.cc:492] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.112413 29037 ts_tablet_manager.cc:1398] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:14:21.112777 29037 tablet_bootstrap.cc:492] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.114549 29037 tablet_bootstrap.cc:654] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.116714 29038 tablet_bootstrap.cc:654] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.118505 29037 tablet_bootstrap.cc:492] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.118656 29037 ts_tablet_manager.cc:1367] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.006s user 0.004s sys 0.001s
I20240502 14:14:21.119017 29039 tablet_bootstrap.cc:492] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.119170 29039 ts_tablet_manager.cc:1367] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.016s user 0.008s sys 0.007s
I20240502 14:14:21.119345 29038 tablet_bootstrap.cc:492] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.119519 29038 ts_tablet_manager.cc:1367] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.002s
I20240502 14:14:21.119472 29037 raft_consensus.cc:348] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.119851 29039 raft_consensus.cc:348] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.120052 29037 raft_consensus.cc:374] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.120083 29039 raft_consensus.cc:374] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.120179 29037 raft_consensus.cc:729] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.120219 29039 raft_consensus.cc:729] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.120445 29037 consensus_queue.cc:260] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.120489 29038 raft_consensus.cc:348] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.120501 29039 consensus_queue.cc:260] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.120685 29038 raft_consensus.cc:374] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.121057 29037 ts_tablet_manager.cc:1398] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:14:21.121251 29039 ts_tablet_manager.cc:1398] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:14:21.121394 29037 tablet_bootstrap.cc:492] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.120795 29038 raft_consensus.cc:729] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.121583 29039 tablet_bootstrap.cc:492] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.121852 29038 consensus_queue.cc:260] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.122540 29038 ts_tablet_manager.cc:1398] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 14:14:21.122900 29038 tablet_bootstrap.cc:492] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.123093 29037 tablet_bootstrap.cc:654] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.123529 29039 tablet_bootstrap.cc:654] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.125030 29038 tablet_bootstrap.cc:654] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.131925 29039 tablet_bootstrap.cc:492] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.132069 29039 ts_tablet_manager.cc:1367] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.006s
I20240502 14:14:21.132334 29037 tablet_bootstrap.cc:492] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.132495 29037 ts_tablet_manager.cc:1367] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.006s
I20240502 14:14:21.132696 29039 raft_consensus.cc:348] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.132865 29039 raft_consensus.cc:374] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.132928 29039 raft_consensus.cc:729] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.133145 29037 raft_consensus.cc:348] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.133191 29039 consensus_queue.cc:260] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.133332 29037 raft_consensus.cc:374] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.133410 29037 raft_consensus.cc:729] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.133759 29039 ts_tablet_manager.cc:1398] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.133680 29037 consensus_queue.cc:260] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.134285 29037 ts_tablet_manager.cc:1398] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.134330 29039 tablet_bootstrap.cc:492] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.134634 29037 tablet_bootstrap.cc:492] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.134824 29038 tablet_bootstrap.cc:492] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.135025 29038 ts_tablet_manager.cc:1367] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.001s
I20240502 14:14:21.135838 29038 raft_consensus.cc:348] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.136016 29039 tablet_bootstrap.cc:654] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.136191 29037 tablet_bootstrap.cc:654] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.138983 29043 raft_consensus.cc:504] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.139797 29043 leader_election.cc:288] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.140450 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.140626 28974 raft_consensus.cc:3037] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.141659 29043 raft_consensus.cc:482] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.141800 29043 raft_consensus.cc:504] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.142377 29043 leader_election.cc:288] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.143019 28973 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2bcfe7bce7af41b79ed1415e7a8b76ed" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.143317 28974 raft_consensus.cc:2454] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
W20240502 14:14:21.143739 28760 tablet.cc:2369] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:14:21.144279 28649 leader_election.cc:302] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
W20240502 14:14:21.144680 28649 leader_election.cc:341] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.145326 29047 raft_consensus.cc:482] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.145543 29047 raft_consensus.cc:504] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.145802 29043 raft_consensus.cc:2781] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.146045 29043 raft_consensus.cc:686] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Becoming Leader. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:21.146669 29043 consensus_queue.cc:237] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.147938 29047 leader_election.cc:288] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:21.148998 29039 tablet_bootstrap.cc:492] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.149134 29039 ts_tablet_manager.cc:1367] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.007s
I20240502 14:14:21.149986 29039 raft_consensus.cc:348] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.150348 29039 raft_consensus.cc:374] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.150516 29039 raft_consensus.cc:729] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.150930 29039 consensus_queue.cc:260] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.151801 29039 ts_tablet_manager.cc:1398] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:21.152245 29039 tablet_bootstrap.cc:492] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.154402 29039 tablet_bootstrap.cc:654] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.136101 29038 raft_consensus.cc:374] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.140188 28844 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.168016 29038 raft_consensus.cc:729] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.145655 28843 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2bcfe7bce7af41b79ed1415e7a8b76ed" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.168466 29038 consensus_queue.cc:260] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
W20240502 14:14:21.169154 28648 leader_election.cc:341] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.169464 29038 ts_tablet_manager.cc:1398] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.034s user 0.001s sys 0.001s
W20240502 14:14:21.169644 28648 leader_election.cc:341] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.169845 29038 tablet_bootstrap.cc:492] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.169787 28648 leader_election.cc:302] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03; no voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4
I20240502 14:14:21.171028 29062 raft_consensus.cc:2726] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:14:21.174451 29038 tablet_bootstrap.cc:654] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.186089 28843 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f7a111dfadd448bab1d5ffdffd2a9fbc" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.186614 28843 raft_consensus.cc:2454] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.187475 28908 leader_election.cc:302] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.188009 29047 raft_consensus.cc:2781] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.188143 29047 raft_consensus.cc:482] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.188237 29047 raft_consensus.cc:3037] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.191557 29038 tablet_bootstrap.cc:492] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.191707 29039 tablet_bootstrap.cc:492] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.191557 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f7a111dfadd448bab1d5ffdffd2a9fbc" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.191815 29038 ts_tablet_manager.cc:1367] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.022s user 0.006s sys 0.000s
I20240502 14:14:21.192023 29039 ts_tablet_manager.cc:1367] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.040s user 0.016s sys 0.020s
I20240502 14:14:21.192062 28573 catalog_manager.cc:5484] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 reported cstate change: term changed from 0 to 1, leader changed from <none> to c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "c9eb1da45f824ba89668a67f863f6c03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
W20240502 14:14:21.192826 28907 leader_election.cc:341] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.192878 29039 raft_consensus.cc:348] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.193091 29039 raft_consensus.cc:374] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.193179 29039 raft_consensus.cc:729] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.193467 29039 consensus_queue.cc:260] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.193954 29038 raft_consensus.cc:348] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.194315 29038 raft_consensus.cc:374] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.194456 29038 raft_consensus.cc:729] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.194833 29038 consensus_queue.cc:260] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.198146 29039 ts_tablet_manager.cc:1398] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.006s user 0.003s sys 0.003s
I20240502 14:14:21.198459 29039 tablet_bootstrap.cc:492] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.199915 29039 tablet_bootstrap.cc:654] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.201597 29039 tablet_bootstrap.cc:492] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.201746 29039 ts_tablet_manager.cc:1367] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.001s
I20240502 14:14:21.202601 29039 raft_consensus.cc:348] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.202836 29039 raft_consensus.cc:374] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.202945 29039 raft_consensus.cc:729] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.203176 29039 consensus_queue.cc:260] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.203612 29039 ts_tablet_manager.cc:1398] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:14:21.203848 29039 tablet_bootstrap.cc:492] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.205008 29039 tablet_bootstrap.cc:654] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.205931 29038 ts_tablet_manager.cc:1398] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.014s user 0.002s sys 0.000s
I20240502 14:14:21.207129 29038 tablet_bootstrap.cc:492] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.209985 29039 tablet_bootstrap.cc:492] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.210131 29039 ts_tablet_manager.cc:1367] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.006s user 0.005s sys 0.000s
I20240502 14:14:21.210158 29037 tablet_bootstrap.cc:492] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.210377 29037 ts_tablet_manager.cc:1367] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.076s user 0.025s sys 0.047s
I20240502 14:14:21.210740 29039 raft_consensus.cc:348] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.210915 29039 raft_consensus.cc:374] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.210997 29039 raft_consensus.cc:729] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.211107 29037 raft_consensus.cc:348] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.211238 29039 consensus_queue.cc:260] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.211354 29037 raft_consensus.cc:374] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.211509 29037 raft_consensus.cc:729] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.211763 29039 ts_tablet_manager.cc:1398] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:14:21.211726 29037 consensus_queue.cc:260] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.212090 29039 tablet_bootstrap.cc:492] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.212358 29037 ts_tablet_manager.cc:1398] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:14:21.212646 29037 tablet_bootstrap.cc:492] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.213152 29038 tablet_bootstrap.cc:654] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.213694 29039 tablet_bootstrap.cc:654] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.214342 29037 tablet_bootstrap.cc:654] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.216315 29043 raft_consensus.cc:482] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.216552 29043 raft_consensus.cc:504] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.217444 29043 leader_election.cc:288] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.218084 28973 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "46170e7b56224a418997a98a85422e02" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.218214 28843 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "46170e7b56224a418997a98a85422e02" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.218322 28973 raft_consensus.cc:2454] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.218427 28843 raft_consensus.cc:2454] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.218825 28648 leader_election.cc:302] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.221100 29043 raft_consensus.cc:2781] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.221244 29043 raft_consensus.cc:482] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.221360 29043 raft_consensus.cc:3037] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.222988 29039 tablet_bootstrap.cc:492] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.223304 29039 ts_tablet_manager.cc:1367] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.006s
I20240502 14:14:21.224071 29039 raft_consensus.cc:348] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.224341 29039 raft_consensus.cc:374] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.224478 29039 raft_consensus.cc:729] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.224771 29039 consensus_queue.cc:260] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.225478 29039 ts_tablet_manager.cc:1398] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:14:21.225834 29039 tablet_bootstrap.cc:492] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.227674 29039 tablet_bootstrap.cc:654] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.229085 29038 tablet_bootstrap.cc:492] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.229138 29037 tablet_bootstrap.cc:492] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.229355 29038 ts_tablet_manager.cc:1367] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.022s user 0.003s sys 0.004s
I20240502 14:14:21.229373 29037 ts_tablet_manager.cc:1367] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.017s user 0.007s sys 0.007s
I20240502 14:14:21.230271 29038 raft_consensus.cc:348] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.230541 29038 raft_consensus.cc:374] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.230654 29038 raft_consensus.cc:729] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.230585 29037 raft_consensus.cc:348] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.230914 29038 consensus_queue.cc:260] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.231086 29037 raft_consensus.cc:374] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.231614 29038 ts_tablet_manager.cc:1398] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.232059 29038 tablet_bootstrap.cc:492] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.231223 29037 raft_consensus.cc:729] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.232436 29037 consensus_queue.cc:260] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.233290 29037 ts_tablet_manager.cc:1398] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:14:21.233712 29038 tablet_bootstrap.cc:654] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.233680 29037 tablet_bootstrap.cc:492] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.234283 29048 raft_consensus.cc:482] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.234432 29048 raft_consensus.cc:504] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.236347 29037 tablet_bootstrap.cc:654] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.236580 29048 leader_election.cc:288] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.240049 29047 raft_consensus.cc:504] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.241055 29047 leader_election.cc:288] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:21.241566 29043 raft_consensus.cc:504] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.242468 29043 leader_election.cc:288] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.243216 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f7a111dfadd448bab1d5ffdffd2a9fbc" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
W20240502 14:14:21.243832 28907 leader_election.cc:341] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.244118 28843 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f7a111dfadd448bab1d5ffdffd2a9fbc" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.244451 28843 raft_consensus.cc:3037] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.248942 28973 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "46170e7b56224a418997a98a85422e02" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.249261 28973 raft_consensus.cc:3037] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
W20240502 14:14:21.253998 29020 tablet.cc:2369] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:14:21.263479 29039 tablet_bootstrap.cc:492] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.263762 29039 ts_tablet_manager.cc:1367] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.038s user 0.005s sys 0.017s
I20240502 14:14:21.264889 29039 raft_consensus.cc:348] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.265230 29039 raft_consensus.cc:374] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.265373 29039 raft_consensus.cc:729] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.265805 29039 consensus_queue.cc:260] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.266757 29039 ts_tablet_manager.cc:1398] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:21.267231 29039 tablet_bootstrap.cc:492] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.269666 29039 tablet_bootstrap.cc:654] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.271591 29043 raft_consensus.cc:482] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.271804 29043 raft_consensus.cc:504] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.272581 28844 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "46170e7b56224a418997a98a85422e02" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.272781 28844 raft_consensus.cc:3037] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.272786 29043 leader_election.cc:288] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.273715 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "222aabe6414f498fa3749f2f83873903" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.274075 29038 tablet_bootstrap.cc:492] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.274237 29038 ts_tablet_manager.cc:1367] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.042s user 0.007s sys 0.003s
W20240502 14:14:21.274672 28649 leader_election.cc:341] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.274832 29038 raft_consensus.cc:348] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.275053 29038 raft_consensus.cc:374] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.275135 29038 raft_consensus.cc:729] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.275408 29038 consensus_queue.cc:260] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.276008 29038 ts_tablet_manager.cc:1398] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:14:21.276515 29038 tablet_bootstrap.cc:492] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.278261 29038 tablet_bootstrap.cc:654] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.280208 29043 raft_consensus.cc:482] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.280387 29043 raft_consensus.cc:504] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.280998 29043 leader_election.cc:288] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.281661 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "298bfef2fe6c4a9593e3f795ae6753a0" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
W20240502 14:14:21.282214 28649 leader_election.cc:341] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.283555 29043 raft_consensus.cc:482] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.283720 29043 raft_consensus.cc:504] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.284392 29043 leader_election.cc:288] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.285126 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8234addac59a4ca299a252c1725e7333" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.285377 28974 raft_consensus.cc:2454] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.285903 28649 leader_election.cc:302] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.286293 29043 raft_consensus.cc:2781] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.286409 29043 raft_consensus.cc:482] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.286523 29043 raft_consensus.cc:3037] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.291445 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc33ffa96d0d4b498f44531cb67c342c" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.291695 28714 raft_consensus.cc:2454] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.278772 28842 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "222aabe6414f498fa3749f2f83873903" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.292546 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc33ffa96d0d4b498f44531cb67c342c" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.281649 28841 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "298bfef2fe6c4a9593e3f795ae6753a0" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.292765 28974 raft_consensus.cc:2454] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.292893 28841 raft_consensus.cc:2454] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.289500 28840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8234addac59a4ca299a252c1725e7333" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
W20240502 14:14:21.293278 28648 leader_election.cc:341] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.292320 28777 leader_election.cc:302] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters:
I20240502 14:14:21.293429 28648 leader_election.cc:302] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03; no voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4
I20240502 14:14:21.293645 29048 raft_consensus.cc:2781] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.293756 29048 raft_consensus.cc:482] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.293846 29048 raft_consensus.cc:3037] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.293833 29062 raft_consensus.cc:2726] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:14:21.294668 28648 leader_election.cc:302] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters: f14e011f333f4dbeb909c21e11b48ea4
W20240502 14:14:21.295037 28648 leader_election.cc:341] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.295318 29062 raft_consensus.cc:2781] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.295404 29062 raft_consensus.cc:482] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.295529 29062 raft_consensus.cc:3037] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.298359 29037 tablet_bootstrap.cc:492] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.298518 29037 ts_tablet_manager.cc:1367] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.065s user 0.013s sys 0.049s
I20240502 14:14:21.299320 29039 tablet_bootstrap.cc:492] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.299335 29037 raft_consensus.cc:348] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.299551 29039 ts_tablet_manager.cc:1367] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.032s user 0.008s sys 0.020s
I20240502 14:14:21.299597 29037 raft_consensus.cc:374] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.299767 29037 raft_consensus.cc:729] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.300159 29038 tablet_bootstrap.cc:492] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.300127 29037 consensus_queue.cc:260] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.300431 29038 ts_tablet_manager.cc:1367] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.024s user 0.002s sys 0.002s
I20240502 14:14:21.300421 29039 raft_consensus.cc:348] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.300753 29039 raft_consensus.cc:374] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.300923 29039 raft_consensus.cc:729] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.301288 29039 consensus_queue.cc:260] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.301489 29038 raft_consensus.cc:348] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.301860 29038 raft_consensus.cc:374] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.302019 29038 raft_consensus.cc:729] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.302414 29038 consensus_queue.cc:260] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.305832 29038 ts_tablet_manager.cc:1398] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.005s user 0.000s sys 0.004s
I20240502 14:14:21.306061 29039 ts_tablet_manager.cc:1398] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240502 14:14:21.309288 29038 tablet_bootstrap.cc:492] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.311115 29039 tablet_bootstrap.cc:492] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.311445 29038 tablet_bootstrap.cc:654] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.313007 29087 raft_consensus.cc:482] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.313174 29087 raft_consensus.cc:504] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.313669 29039 tablet_bootstrap.cc:654] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.314790 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207e9900d5d34ccda396e881a0134ea2" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.315093 28974 raft_consensus.cc:2454] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.315718 28649 leader_election.cc:302] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.316135 29037 ts_tablet_manager.cc:1398] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.017s user 0.007s sys 0.011s
I20240502 14:14:21.316383 29089 raft_consensus.cc:2781] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.316419 28840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207e9900d5d34ccda396e881a0134ea2" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.316604 29037 tablet_bootstrap.cc:492] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.316707 29089 raft_consensus.cc:482] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.317315 29038 tablet_bootstrap.cc:492] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.317553 29038 ts_tablet_manager.cc:1367] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.008s user 0.005s sys 0.001s
I20240502 14:14:21.317894 29089 raft_consensus.cc:3037] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
W20240502 14:14:21.318498 28648 leader_election.cc:341] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:21.318435 29038 raft_consensus.cc:348] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.318637 29038 raft_consensus.cc:374] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.318715 29038 raft_consensus.cc:729] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.318825 29087 leader_election.cc:288] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.319036 29038 consensus_queue.cc:260] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.319278 29037 tablet_bootstrap.cc:654] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.319662 29038 ts_tablet_manager.cc:1398] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:14:21.319996 29038 tablet_bootstrap.cc:492] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.321079 29039 tablet_bootstrap.cc:492] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.321290 29039 ts_tablet_manager.cc:1367] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.005s
I20240502 14:14:21.321431 29038 tablet_bootstrap.cc:654] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.322384 29039 raft_consensus.cc:348] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.322804 29039 raft_consensus.cc:374] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.322960 29039 raft_consensus.cc:729] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.323349 29039 consensus_queue.cc:260] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.324575 29038 tablet_bootstrap.cc:492] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.324707 29039 ts_tablet_manager.cc:1398] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240502 14:14:21.324729 29038 ts_tablet_manager.cc:1367] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.001s
I20240502 14:14:21.325050 29039 tablet_bootstrap.cc:492] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.325278 29038 raft_consensus.cc:348] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.325443 29038 raft_consensus.cc:374] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.325593 29038 raft_consensus.cc:729] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.325794 29038 consensus_queue.cc:260] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.326630 29038 ts_tablet_manager.cc:1398] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:14:21.326646 29037 tablet_bootstrap.cc:492] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.326844 29037 ts_tablet_manager.cc:1367] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.003s
I20240502 14:14:21.326952 29038 tablet_bootstrap.cc:492] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.327564 29039 tablet_bootstrap.cc:654] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.327708 29037 raft_consensus.cc:348] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.328012 29037 raft_consensus.cc:374] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.328152 29037 raft_consensus.cc:729] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.328706 29038 tablet_bootstrap.cc:654] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.328486 29037 consensus_queue.cc:260] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.330060 29037 ts_tablet_manager.cc:1398] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 14:14:21.330561 29037 tablet_bootstrap.cc:492] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03: Bootstrap starting.
I20240502 14:14:21.332935 29037 tablet_bootstrap.cc:654] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.334045 29038 tablet_bootstrap.cc:492] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.334159 29038 ts_tablet_manager.cc:1367] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.005s
I20240502 14:14:21.334841 29038 raft_consensus.cc:348] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.335742 29039 tablet_bootstrap.cc:492] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.335783 29038 raft_consensus.cc:374] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.335863 29038 raft_consensus.cc:729] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.335922 29039 ts_tablet_manager.cc:1367] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.003s
I20240502 14:14:21.336149 29038 consensus_queue.cc:260] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.336280 29087 raft_consensus.cc:482] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.336490 29087 raft_consensus.cc:504] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.336659 29038 ts_tablet_manager.cc:1398] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:14:21.336957 29038 tablet_bootstrap.cc:492] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.336885 29039 raft_consensus.cc:348] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.337182 29039 raft_consensus.cc:374] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.337339 29039 raft_consensus.cc:729] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.337702 29039 consensus_queue.cc:260] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.337824 28840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ab370a1503e141c0a807d33964687fc9" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.338073 28840 raft_consensus.cc:2454] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.338596 29039 ts_tablet_manager.cc:1398] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.338603 28648 leader_election.cc:302] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters:
I20240502 14:14:21.338917 29039 tablet_bootstrap.cc:492] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.339299 29038 tablet_bootstrap.cc:654] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.340094 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ab370a1503e141c0a807d33964687fc9" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.340329 28974 raft_consensus.cc:2454] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.340692 29098 raft_consensus.cc:2781] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.340746 29039 tablet_bootstrap.cc:654] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.340866 29098 raft_consensus.cc:482] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.340981 29087 leader_election.cc:288] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.341383 28843 raft_consensus.cc:2454] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.341370 29043 raft_consensus.cc:504] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.341846 28973 raft_consensus.cc:2454] T 46170e7b56224a418997a98a85422e02 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.341948 29043 leader_election.cc:288] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.344306 29062 raft_consensus.cc:504] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.344782 29062 leader_election.cc:288] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.341142 29098 raft_consensus.cc:3037] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.349046 28908 leader_election.cc:302] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters: c9eb1da45f824ba89668a67f863f6c03
I20240502 14:14:21.349725 29047 raft_consensus.cc:2781] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.349776 29038 tablet_bootstrap.cc:492] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.349907 29038 ts_tablet_manager.cc:1367] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.013s user 0.005s sys 0.001s
I20240502 14:14:21.349915 29047 raft_consensus.cc:686] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [term 1 LEADER]: Becoming Leader. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Running, Role: LEADER
I20240502 14:14:21.350631 29038 raft_consensus.cc:348] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.350570 29047 consensus_queue.cc:237] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.350844 29038 raft_consensus.cc:374] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.350931 29038 raft_consensus.cc:729] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.351132 29038 consensus_queue.cc:260] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.351649 29038 ts_tablet_manager.cc:1398] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:14:21.351920 29038 tablet_bootstrap.cc:492] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.353550 29038 tablet_bootstrap.cc:654] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.354635 28649 leader_election.cc:302] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.354895 29062 raft_consensus.cc:2781] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.355041 29062 raft_consensus.cc:686] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Becoming Leader. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:21.355495 29039 tablet_bootstrap.cc:492] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.355441 28843 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8234addac59a4ca299a252c1725e7333" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.355648 28843 raft_consensus.cc:3037] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.355670 29039 ts_tablet_manager.cc:1367] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.011s
I20240502 14:14:21.355672 29062 consensus_queue.cc:237] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.356256 28973 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8234addac59a4ca299a252c1725e7333" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.356384 29039 raft_consensus.cc:348] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.357882 29038 tablet_bootstrap.cc:492] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.357903 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "298bfef2fe6c4a9593e3f795ae6753a0" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.358084 29038 ts_tablet_manager.cc:1367] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.003s
I20240502 14:14:21.358350 28840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "298bfef2fe6c4a9593e3f795ae6753a0" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.358570 29039 raft_consensus.cc:374] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.358570 28840 raft_consensus.cc:3037] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.358650 29039 raft_consensus.cc:729] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.358714 29038 raft_consensus.cc:348] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.358991 29038 raft_consensus.cc:374] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.359051 29038 raft_consensus.cc:729] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.359241 28973 raft_consensus.cc:3037] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.359278 29038 consensus_queue.cc:260] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.359637 28974 raft_consensus.cc:3037] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.359747 29038 ts_tablet_manager.cc:1398] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:14:21.359992 29038 tablet_bootstrap.cc:492] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000: Bootstrap starting.
I20240502 14:14:21.361459 29037 tablet_bootstrap.cc:492] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03: No bootstrap required, opened a new log
I20240502 14:14:21.361611 29037 ts_tablet_manager.cc:1367] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03: Time spent bootstrapping tablet: real 0.031s user 0.002s sys 0.002s
I20240502 14:14:21.361722 29090 raft_consensus.cc:482] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.361805 29038 tablet_bootstrap.cc:654] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.361888 29090 raft_consensus.cc:504] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.362643 29037 raft_consensus.cc:348] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.362854 29037 raft_consensus.cc:374] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.362952 29037 raft_consensus.cc:729] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.362937 29090 leader_election.cc:288] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.363217 29037 consensus_queue.cc:260] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.359217 29039 consensus_queue.cc:260] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.363787 29037 ts_tablet_manager.cc:1398] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.364610 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207e9900d5d34ccda396e881a0134ea2" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.364607 28972 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207e9900d5d34ccda396e881a0134ea2" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.365399 28972 raft_consensus.cc:2454] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.365568 29039 ts_tablet_manager.cc:1398] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.010s user 0.002s sys 0.001s
I20240502 14:14:21.366358 28779 leader_election.cc:302] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.366775 29039 tablet_bootstrap.cc:492] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4: Bootstrap starting.
I20240502 14:14:21.368544 29090 raft_consensus.cc:2781] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.368716 29090 raft_consensus.cc:482] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.368832 29090 raft_consensus.cc:3037] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.369043 29105 raft_consensus.cc:482] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.369288 29105 raft_consensus.cc:504] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.371402 28713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2b2fd9b7b3c43928513bfe289b87e80" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.371686 28713 raft_consensus.cc:2454] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.372270 28777 leader_election.cc:302] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters:
I20240502 14:14:21.372332 29039 tablet_bootstrap.cc:654] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:21.372881 29105 leader_election.cc:288] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.373143 29105 raft_consensus.cc:2781] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.373713 29105 raft_consensus.cc:482] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.373883 29105 raft_consensus.cc:3037] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.374682 28572 catalog_manager.cc:5484] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 reported cstate change: term changed from 0 to 1, leader changed from <none> to f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "f14e011f333f4dbeb909c21e11b48ea4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:21.381755 28972 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2b2fd9b7b3c43928513bfe289b87e80" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.382063 28972 raft_consensus.cc:2454] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.387563 29038 tablet_bootstrap.cc:492] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000: No bootstrap required, opened a new log
I20240502 14:14:21.387735 29038 ts_tablet_manager.cc:1367] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000: Time spent bootstrapping tablet: real 0.028s user 0.014s sys 0.012s
I20240502 14:14:21.388649 29038 raft_consensus.cc:348] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.388870 29038 raft_consensus.cc:374] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.388972 29038 raft_consensus.cc:729] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.388736 28572 catalog_manager.cc:5484] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 reported cstate change: term changed from 0 to 1, leader changed from <none> to c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "c9eb1da45f824ba89668a67f863f6c03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.389214 29038 consensus_queue.cc:260] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.389812 29038 ts_tablet_manager.cc:1398] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:21.390911 29039 tablet_bootstrap.cc:492] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4: No bootstrap required, opened a new log
I20240502 14:14:21.391078 29039 ts_tablet_manager.cc:1367] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4: Time spent bootstrapping tablet: real 0.025s user 0.010s sys 0.001s
I20240502 14:14:21.391979 29039 raft_consensus.cc:348] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.392230 29039 raft_consensus.cc:374] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:21.392925 29039 raft_consensus.cc:729] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Initialized, Role: FOLLOWER
I20240502 14:14:21.393198 29039 consensus_queue.cc:260] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.393929 29039 ts_tablet_manager.cc:1398] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:21.399048 29047 raft_consensus.cc:482] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.399191 29047 raft_consensus.cc:504] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.399713 29047 leader_election.cc:288] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:21.400166 28713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "840e603053f94d7192f26b0dfda184f6" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.400208 28841 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "840e603053f94d7192f26b0dfda184f6" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.400390 28841 raft_consensus.cc:2454] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.400390 28713 raft_consensus.cc:2454] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.400763 28908 leader_election.cc:302] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.401039 29047 raft_consensus.cc:2781] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.401137 29047 raft_consensus.cc:482] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.401247 29047 raft_consensus.cc:3037] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.404026 29106 raft_consensus.cc:482] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.404155 29106 raft_consensus.cc:504] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.404798 29106 leader_election.cc:288] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.405189 28972 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e29f784041244d6906d011a85b5725a" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.405191 28713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e29f784041244d6906d011a85b5725a" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.405350 28972 raft_consensus.cc:2454] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.405364 28713 raft_consensus.cc:2454] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.405692 28779 leader_election.cc:302] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.405928 29106 raft_consensus.cc:2781] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.406020 29106 raft_consensus.cc:482] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.406113 29106 raft_consensus.cc:3037] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.438122 28844 raft_consensus.cc:2454] T 46170e7b56224a418997a98a85422e02 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.438122 28973 raft_consensus.cc:2454] T 8234addac59a4ca299a252c1725e7333 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.438131 29089 raft_consensus.cc:504] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.438628 28974 raft_consensus.cc:2454] T 298bfef2fe6c4a9593e3f795ae6753a0 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.439069 29089 leader_election.cc:288] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.439194 28649 leader_election.cc:302] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.439548 28649 leader_election.cc:302] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.439433 29098 raft_consensus.cc:504] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.440138 29098 leader_election.cc:288] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.440295 29098 raft_consensus.cc:2781] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.440284 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207e9900d5d34ccda396e881a0134ea2" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.440434 29098 raft_consensus.cc:686] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Becoming Leader. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:21.440474 28974 raft_consensus.cc:3037] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.440726 29098 consensus_queue.cc:237] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.441030 29048 raft_consensus.cc:504] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.441690 29075 raft_consensus.cc:482] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.441841 29075 raft_consensus.cc:504] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.442299 28714 raft_consensus.cc:2379] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d3ff4b86b3e846c9ba04b989f823a000 in current term 1: Already voted for candidate c9eb1da45f824ba89668a67f863f6c03 in this term.
I20240502 14:14:21.442404 28973 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ab370a1503e141c0a807d33964687fc9" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.442615 28973 raft_consensus.cc:3037] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.442780 29075 leader_election.cc:288] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:21.443646 29087 raft_consensus.cc:2781] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.443806 29087 raft_consensus.cc:686] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Becoming Leader. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:21.444185 29087 consensus_queue.cc:237] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.445310 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fec9c54b0b9041eeab5a4a45dbdf1a6e" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.445506 28714 raft_consensus.cc:2454] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.445960 28907 leader_election.cc:302] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.446388 29075 raft_consensus.cc:2781] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.446511 29075 raft_consensus.cc:482] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.446591 29075 raft_consensus.cc:3037] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.447003 29087 raft_consensus.cc:482] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.447149 29087 raft_consensus.cc:504] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.447669 28841 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207e9900d5d34ccda396e881a0134ea2" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.447751 29087 leader_election.cc:288] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.448100 28842 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ab370a1503e141c0a807d33964687fc9" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.448284 28972 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1a7393c68b047b2a01e26455d8b27ae" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.448330 28842 raft_consensus.cc:3037] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.448472 28972 raft_consensus.cc:2454] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.448869 28649 leader_election.cc:302] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.446419 28573 catalog_manager.cc:5484] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 reported cstate change: term changed from 0 to 1, leader changed from <none> to c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "c9eb1da45f824ba89668a67f863f6c03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.449270 29087 raft_consensus.cc:2781] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.449357 29087 raft_consensus.cc:482] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.449455 29087 raft_consensus.cc:3037] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.450311 29048 leader_election.cc:288] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.450338 28839 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1a7393c68b047b2a01e26455d8b27ae" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.450718 28839 raft_consensus.cc:2454] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 0.
I20240502 14:14:21.451063 28972 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc33ffa96d0d4b498f44531cb67c342c" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.450547 29048 raft_consensus.cc:482] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.451259 28972 raft_consensus.cc:3037] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.451303 29048 raft_consensus.cc:504] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.453228 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bc33ffa96d0d4b498f44531cb67c342c" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.453425 28714 raft_consensus.cc:3037] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.454746 28573 catalog_manager.cc:5484] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 reported cstate change: term changed from 0 to 1, leader changed from <none> to c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "c9eb1da45f824ba89668a67f863f6c03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.453227 28844 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fec9c54b0b9041eeab5a4a45dbdf1a6e" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.455770 28844 raft_consensus.cc:2454] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.457715 28713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb281e01d01f457f84364f0b432a676e" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.457913 28713 raft_consensus.cc:2454] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.458400 28777 leader_election.cc:302] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters:
I20240502 14:14:21.458551 29048 leader_election.cc:288] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.458690 29048 raft_consensus.cc:2781] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.458801 29048 raft_consensus.cc:482] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.458954 29048 raft_consensus.cc:3037] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.459046 28971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb281e01d01f457f84364f0b432a676e" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.459340 28971 raft_consensus.cc:2454] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 0.
I20240502 14:14:21.472148 29062 raft_consensus.cc:482] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.472296 29062 raft_consensus.cc:504] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.472918 29062 leader_election.cc:288] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.473327 28844 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f7a111dfadd448bab1d5ffdffd2a9fbc" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.473407 28971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f7a111dfadd448bab1d5ffdffd2a9fbc" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.473664 28844 raft_consensus.cc:2379] T f7a111dfadd448bab1d5ffdffd2a9fbc P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c9eb1da45f824ba89668a67f863f6c03 in current term 1: Already voted for candidate f14e011f333f4dbeb909c21e11b48ea4 in this term.
I20240502 14:14:21.474131 28648 leader_election.cc:302] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03; no voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4
I20240502 14:14:21.474395 29062 raft_consensus.cc:3037] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.533146 28973 raft_consensus.cc:2454] T ab370a1503e141c0a807d33964687fc9 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.533167 28840 raft_consensus.cc:2454] T 298bfef2fe6c4a9593e3f795ae6753a0 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.533324 29087 raft_consensus.cc:504] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.534945 28840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1a7393c68b047b2a01e26455d8b27ae" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.535171 28840 raft_consensus.cc:3037] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.535617 29047 raft_consensus.cc:504] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.536592 28713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "840e603053f94d7192f26b0dfda184f6" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.536772 28713 raft_consensus.cc:3037] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.536888 28714 raft_consensus.cc:2454] T bc33ffa96d0d4b498f44531cb67c342c P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.537437 29087 leader_election.cc:288] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.537920 28777 leader_election.cc:302] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters:
I20240502 14:14:21.538309 28649 leader_election.cc:302] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.538587 29087 raft_consensus.cc:2781] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.538764 29087 raft_consensus.cc:686] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Becoming Leader. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:21.539019 29087 consensus_queue.cc:237] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.539460 29110 raft_consensus.cc:2781] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.539670 28973 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1a7393c68b047b2a01e26455d8b27ae" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.539834 28973 raft_consensus.cc:3037] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.541150 28844 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "840e603053f94d7192f26b0dfda184f6" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.541373 28844 raft_consensus.cc:3037] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.541741 29047 leader_election.cc:288] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:21.542269 29110 raft_consensus.cc:686] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [term 1 LEADER]: Becoming Leader. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Running, Role: LEADER
I20240502 14:14:21.542825 29110 consensus_queue.cc:237] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.543836 28573 catalog_manager.cc:5484] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 reported cstate change: term changed from 0 to 1, leader changed from <none> to c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "c9eb1da45f824ba89668a67f863f6c03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.548112 28573 catalog_manager.cc:5484] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 reported cstate change: term changed from 0 to 1, leader changed from <none> to d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130). New cstate: current_term: 1 leader_uuid: "d3ff4b86b3e846c9ba04b989f823a000" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.578362 28839 raft_consensus.cc:3037] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.593600 29047 raft_consensus.cc:482] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.593817 29047 raft_consensus.cc:504] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.594437 29047 leader_election.cc:288] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:21.594882 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "222aabe6414f498fa3749f2f83873903" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.594995 28838 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "222aabe6414f498fa3749f2f83873903" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.595135 28714 raft_consensus.cc:2454] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.595278 28838 raft_consensus.cc:2454] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.595491 28907 leader_election.cc:302] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.595741 29047 raft_consensus.cc:2781] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.595826 29047 raft_consensus.cc:482] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.595908 29047 raft_consensus.cc:3037] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.623348 29114 raft_consensus.cc:482] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.623565 29114 raft_consensus.cc:504] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.624289 29114 leader_election.cc:288] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477)
I20240502 14:14:21.625342 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2bcfe7bce7af41b79ed1415e7a8b76ed" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.625612 28714 raft_consensus.cc:2454] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.625818 28838 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2bcfe7bce7af41b79ed1415e7a8b76ed" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:21.626140 28907 leader_election.cc:302] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.626420 29114 raft_consensus.cc:2781] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:21.626539 29114 raft_consensus.cc:482] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.626629 29114 raft_consensus.cc:3037] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.626629 28838 raft_consensus.cc:2454] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 0.
I20240502 14:14:21.628505 28843 raft_consensus.cc:2454] T 8234addac59a4ca299a252c1725e7333 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.628516 29075 raft_consensus.cc:504] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.629065 29075 leader_election.cc:288] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:21.629565 28843 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fec9c54b0b9041eeab5a4a45dbdf1a6e" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.629751 28974 raft_consensus.cc:2454] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.629793 28843 raft_consensus.cc:3037] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.632126 29062 raft_consensus.cc:2726] T f7a111dfadd448bab1d5ffdffd2a9fbc P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:14:21.632642 28649 leader_election.cc:302] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.629442 29090 raft_consensus.cc:504] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.632979 28841 raft_consensus.cc:2379] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate c9eb1da45f824ba89668a67f863f6c03 in current term 1: Already voted for candidate d3ff4b86b3e846c9ba04b989f823a000 in this term.
I20240502 14:14:21.632961 29062 raft_consensus.cc:2781] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.633126 29062 raft_consensus.cc:686] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Becoming Leader. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:21.633302 29090 leader_election.cc:288] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.633394 29062 consensus_queue.cc:237] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.634032 28712 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fec9c54b0b9041eeab5a4a45dbdf1a6e" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.634258 28712 raft_consensus.cc:3037] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.634773 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207e9900d5d34ccda396e881a0134ea2" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.635021 28974 raft_consensus.cc:2379] T 207e9900d5d34ccda396e881a0134ea2 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate d3ff4b86b3e846c9ba04b989f823a000 in current term 1: Already voted for candidate c9eb1da45f824ba89668a67f863f6c03 in this term.
I20240502 14:14:21.635463 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "207e9900d5d34ccda396e881a0134ea2" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.635973 28777 leader_election.cc:302] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000; no voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4
I20240502 14:14:21.636369 29090 raft_consensus.cc:2726] T 207e9900d5d34ccda396e881a0134ea2 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 14:14:21.636272 28573 catalog_manager.cc:5484] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 reported cstate change: term changed from 0 to 1, leader changed from <none> to c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "c9eb1da45f824ba89668a67f863f6c03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.729811 28843 raft_consensus.cc:2454] T fec9c54b0b9041eeab5a4a45dbdf1a6e P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.729815 28713 raft_consensus.cc:2454] T 840e603053f94d7192f26b0dfda184f6 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.729815 28972 raft_consensus.cc:2454] T bc33ffa96d0d4b498f44531cb67c342c P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.729960 29106 raft_consensus.cc:504] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.730361 29105 raft_consensus.cc:504] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.730530 28908 leader_election.cc:302] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.730557 28973 raft_consensus.cc:2454] T a1a7393c68b047b2a01e26455d8b27ae P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.730734 29106 leader_election.cc:288] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.730993 29105 leader_election.cc:288] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.731149 28907 leader_election.cc:302] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.733505 28712 raft_consensus.cc:2454] T fec9c54b0b9041eeab5a4a45dbdf1a6e P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.733659 29047 raft_consensus.cc:504] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.733808 28842 raft_consensus.cc:2454] T ab370a1503e141c0a807d33964687fc9 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.734279 28840 raft_consensus.cc:2454] T a1a7393c68b047b2a01e26455d8b27ae P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 1.
I20240502 14:14:21.734360 29047 leader_election.cc:288] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255)
I20240502 14:14:21.734536 29047 raft_consensus.cc:2781] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.734664 28973 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e29f784041244d6906d011a85b5725a" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.734690 29047 raft_consensus.cc:686] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [term 1 LEADER]: Becoming Leader. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Running, Role: LEADER
I20240502 14:14:21.734874 28973 raft_consensus.cc:3037] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.735121 29047 consensus_queue.cc:237] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.735489 28840 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "222aabe6414f498fa3749f2f83873903" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.735673 28840 raft_consensus.cc:3037] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.737159 28844 raft_consensus.cc:2454] T 840e603053f94d7192f26b0dfda184f6 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.737795 28649 leader_election.cc:302] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.738133 29062 raft_consensus.cc:2781] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.738019 29048 raft_consensus.cc:504] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.738294 29062 raft_consensus.cc:686] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Becoming Leader. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:21.738579 29062 consensus_queue.cc:237] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.738957 29116 raft_consensus.cc:2781] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.740648 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb281e01d01f457f84364f0b432a676e" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.740900 28974 raft_consensus.cc:3037] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.742520 28714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bb281e01d01f457f84364f0b432a676e" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.742796 28714 raft_consensus.cc:3037] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.744369 28713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2b2fd9b7b3c43928513bfe289b87e80" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.744607 28713 raft_consensus.cc:3037] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.747290 28573 catalog_manager.cc:5484] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 reported cstate change: term changed from 0 to 1, leader changed from <none> to c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "c9eb1da45f824ba89668a67f863f6c03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.748139 28713 raft_consensus.cc:2454] T f2b2fd9b7b3c43928513bfe289b87e80 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.738638 29048 leader_election.cc:288] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.748724 29105 raft_consensus.cc:482] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:21.749127 29062 consensus_queue.cc:1035] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.749276 29105 raft_consensus.cc:504] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.749439 29087 consensus_queue.cc:1035] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.750067 29105 leader_election.cc:288] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.738333 28840 raft_consensus.cc:2454] T 222aabe6414f498fa3749f2f83873903 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.751001 28908 leader_election.cc:302] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.751749 28971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:21.752008 28971 raft_consensus.cc:2454] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.752349 29062 consensus_queue.cc:1035] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.752918 29047 raft_consensus.cc:2781] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.753029 28779 leader_election.cc:302] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.753309 29105 raft_consensus.cc:2781] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20240502 14:14:21.753445 29105 raft_consensus.cc:482] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:21.753578 29105 raft_consensus.cc:3037] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:14:21.754691 28777 leader_election.cc:302] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters:
I20240502 14:14:21.754962 29048 raft_consensus.cc:2781] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.755084 29048 raft_consensus.cc:686] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [term 1 LEADER]: Becoming Leader. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Running, Role: LEADER
I20240502 14:14:21.755373 29048 consensus_queue.cc:237] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.756378 28711 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03" is_pre_election: true
I20240502 14:14:21.757301 28974 raft_consensus.cc:2454] T bb281e01d01f457f84364f0b432a676e P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.757836 28779 leader_election.cc:302] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.758136 29106 raft_consensus.cc:2781] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.758273 29106 raft_consensus.cc:686] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [term 1 LEADER]: Becoming Leader. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Running, Role: LEADER
I20240502 14:14:21.758270 28972 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f2b2fd9b7b3c43928513bfe289b87e80" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.758494 28972 raft_consensus.cc:3037] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.758445 29105 raft_consensus.cc:504] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.758527 29106 consensus_queue.cc:237] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.759263 29105 leader_election.cc:288] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 2 election: Requested vote from peers c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:21.759583 28713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "222aabe6414f498fa3749f2f83873903" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.759878 28713 raft_consensus.cc:3037] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.761793 28972 raft_consensus.cc:2454] T f2b2fd9b7b3c43928513bfe289b87e80 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.762405 28711 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.763739 28713 raft_consensus.cc:2454] T 222aabe6414f498fa3749f2f83873903 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.753136 29047 raft_consensus.cc:686] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 LEADER]: Becoming Leader. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Running, Role: LEADER
I20240502 14:14:21.764861 28714 raft_consensus.cc:2454] T bb281e01d01f457f84364f0b432a676e P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.764879 29047 consensus_queue.cc:237] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.766532 28712 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e29f784041244d6906d011a85b5725a" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.766855 28712 raft_consensus.cc:3037] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.768983 28973 raft_consensus.cc:2454] T 3e29f784041244d6906d011a85b5725a P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.769634 28779 leader_election.cc:302] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.769757 28839 raft_consensus.cc:1226] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 2 FOLLOWER]: Rejecting Update request from peer c9eb1da45f824ba89668a67f863f6c03 for earlier term 1. Current term is 2. Ops: [1.1-1.1]
I20240502 14:14:21.769976 29048 raft_consensus.cc:2781] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.770139 29048 raft_consensus.cc:686] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [term 1 LEADER]: Becoming Leader. State: Replica: d3ff4b86b3e846c9ba04b989f823a000, State: Running, Role: LEADER
I20240502 14:14:21.770391 29048 consensus_queue.cc:237] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.770781 29062 consensus_queue.cc:1046] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Peer responded invalid term: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: INVALID_TERM, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.771163 29062 raft_consensus.cc:3032] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:14:21.771270 29062 raft_consensus.cc:729] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:21.771649 29062 consensus_queue.cc:260] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 1.1, Last appended by leader: 1, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.772311 29062 raft_consensus.cc:3037] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:14:21.772377 29116 raft_consensus.cc:686] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 LEADER]: Becoming Leader. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Running, Role: LEADER
I20240502 14:14:21.772794 29116 consensus_queue.cc:237] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.776818 28974 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "d3ff4b86b3e846c9ba04b989f823a000" candidate_term: 2 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:21.777668 28779 leader_election.cc:302] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [CANDIDATE]: Term 2 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d3ff4b86b3e846c9ba04b989f823a000; no voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4
I20240502 14:14:21.778061 29105 raft_consensus.cc:2726] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 2 FOLLOWER]: Leader election lost for term 2. Reason: could not achieve majority
I20240502 14:14:21.782337 28712 raft_consensus.cc:2454] T 3e29f784041244d6906d011a85b5725a P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d3ff4b86b3e846c9ba04b989f823a000 in term 1.
I20240502 14:14:21.782281 29118 consensus_queue.cc:1035] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.782411 28573 catalog_manager.cc:5484] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 reported cstate change: term changed from 0 to 1, leader changed from <none> to f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "f14e011f333f4dbeb909c21e11b48ea4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:21.783020 28573 catalog_manager.cc:5484] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 reported cstate change: term changed from 0 to 1, leader changed from <none> to f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "f14e011f333f4dbeb909c21e11b48ea4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:21.789634 29114 raft_consensus.cc:504] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.790886 28839 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2bcfe7bce7af41b79ed1415e7a8b76ed" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:21.790886 28712 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2bcfe7bce7af41b79ed1415e7a8b76ed" candidate_uuid: "f14e011f333f4dbeb909c21e11b48ea4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c9eb1da45f824ba89668a67f863f6c03"
I20240502 14:14:21.791080 28712 raft_consensus.cc:3037] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.791080 28839 raft_consensus.cc:3037] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:21.794131 28712 raft_consensus.cc:2454] T 2bcfe7bce7af41b79ed1415e7a8b76ed P c9eb1da45f824ba89668a67f863f6c03 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.794540 28907 leader_election.cc:302] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, f14e011f333f4dbeb909c21e11b48ea4; no voters:
I20240502 14:14:21.794852 29116 raft_consensus.cc:2781] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:21.795018 29116 raft_consensus.cc:686] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [term 1 LEADER]: Becoming Leader. State: Replica: f14e011f333f4dbeb909c21e11b48ea4, State: Running, Role: LEADER
I20240502 14:14:21.795107 28839 raft_consensus.cc:2454] T 2bcfe7bce7af41b79ed1415e7a8b76ed P d3ff4b86b3e846c9ba04b989f823a000 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate f14e011f333f4dbeb909c21e11b48ea4 in term 1.
I20240502 14:14:21.795266 29116 consensus_queue.cc:237] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:21.801175 29114 leader_election.cc:288] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [CANDIDATE]: Term 1 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477)
I20240502 14:14:21.804189 28572 catalog_manager.cc:5484] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 reported cstate change: term changed from 0 to 1, leader changed from <none> to f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "f14e011f333f4dbeb909c21e11b48ea4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:21.807224 28573 catalog_manager.cc:5484] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 reported cstate change: term changed from 0 to 1, leader changed from <none> to d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130). New cstate: current_term: 1 leader_uuid: "d3ff4b86b3e846c9ba04b989f823a000" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.811206 28573 catalog_manager.cc:5484] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 reported cstate change: term changed from 0 to 1, leader changed from <none> to d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130). New cstate: current_term: 1 leader_uuid: "d3ff4b86b3e846c9ba04b989f823a000" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.811995 28573 catalog_manager.cc:5484] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 reported cstate change: term changed from 0 to 1, leader changed from <none> to d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130). New cstate: current_term: 1 leader_uuid: "d3ff4b86b3e846c9ba04b989f823a000" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:21.824044 29098 consensus_queue.cc:1035] T 46170e7b56224a418997a98a85422e02 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.825435 28573 catalog_manager.cc:5484] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 reported cstate change: term changed from 0 to 1, leader changed from <none> to f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "f14e011f333f4dbeb909c21e11b48ea4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:21.826429 29114 consensus_queue.cc:1035] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.889912 29098 consensus_queue.cc:1035] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.894235 29043 consensus_queue.cc:1035] T 298bfef2fe6c4a9593e3f795ae6753a0 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.917002 29043 consensus_queue.cc:1035] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.921121 29098 consensus_queue.cc:1035] T 8234addac59a4ca299a252c1725e7333 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.930344 29048 consensus_queue.cc:1035] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.934602 29106 consensus_queue.cc:1035] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.983369 29098 consensus_queue.cc:1035] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:21.985612 29098 consensus_queue.cc:1035] T ab370a1503e141c0a807d33964687fc9 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.030622 29087 consensus_queue.cc:1035] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.036190 29098 consensus_queue.cc:1035] T 207e9900d5d34ccda396e881a0134ea2 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.158295 29116 consensus_queue.cc:1035] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.164996 29114 consensus_queue.cc:1035] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.165954 29114 consensus_queue.cc:1035] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.182694 29106 consensus_queue.cc:1035] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.185752 29087 consensus_queue.cc:1035] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.186304 29075 consensus_queue.cc:1035] T fec9c54b0b9041eeab5a4a45dbdf1a6e P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.189481 29048 consensus_queue.cc:1035] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
I20240502 14:14:22.193825 29048 consensus_queue.cc:1035] T f2b2fd9b7b3c43928513bfe289b87e80 P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.195830 29075 consensus_queue.cc:1035] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.198464 29087 consensus_queue.cc:1035] T a1a7393c68b047b2a01e26455d8b27ae P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.201212 29090 consensus_queue.cc:1035] T 3e29f784041244d6906d011a85b5725a P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.205011 29116 consensus_queue.cc:1035] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.229995 29075 consensus_queue.cc:1035] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.232929 29075 consensus_queue.cc:1035] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.256664 29090 consensus_queue.cc:1035] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:22.264192 29106 consensus_queue.cc:1035] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:14:22.380934 29142 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:22.381399 29142 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:22.389591 29142 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
W20240502 14:14:22.648850 28673 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53010: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.653909 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53004: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.660378 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39056: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.682492 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57416: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.691689 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39070: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.698539 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53004: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.705667 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39056: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.708374 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57400: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.716284 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53010: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.719383 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57416: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.732036 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53004: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.732159 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39070: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.741461 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39056: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.744071 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57400: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.754510 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57416: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.754595 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53010: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.764511 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39070: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.767050 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53004: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.776310 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57400: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.779559 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39056: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.789359 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53010: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.795104 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57416: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.802930 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39070: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.814158 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53004: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.817497 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57400: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.828969 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39056: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.834815 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53010: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.848240 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57416: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.851433 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39070: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.866636 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53004: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.871147 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57400: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.886833 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39056: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.891386 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53010: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.908885 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57416: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.913137 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39070: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.932443 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53004: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.934736 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57400: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.954797 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39056: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.957003 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53010: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.978812 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57416: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:22.981695 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39070: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:23.004606 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57400: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
W20240502 14:14:23.006191 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53004: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:23.033533 28674 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:53010: Illegal state: replica c9eb1da45f824ba89668a67f863f6c03 is not leader of this config: current role FOLLOWER
W20240502 14:14:23.034118 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39056: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:23.059648 28804 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:39070: Illegal state: replica d3ff4b86b3e846c9ba04b989f823a000 is not leader of this config: current role FOLLOWER
W20240502 14:14:23.061833 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57416: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
I20240502 14:14:23.065361 29062 raft_consensus.cc:482] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:23.065550 29062 raft_consensus.cc:504] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:23.066224 29062 leader_election.cc:288] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:23.066603 28839 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 3 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000" is_pre_election: true
I20240502 14:14:23.066684 28971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 3 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4" is_pre_election: true
I20240502 14:14:23.066871 28839 raft_consensus.cc:2454] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 2.
I20240502 14:14:23.067345 28648 leader_election.cc:302] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters: f14e011f333f4dbeb909c21e11b48ea4
I20240502 14:14:23.067588 29062 raft_consensus.cc:2781] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 2 FOLLOWER]: Leader pre-election won for term 3
I20240502 14:14:23.067708 29062 raft_consensus.cc:482] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 2 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:23.067795 29062 raft_consensus.cc:3037] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 2 FOLLOWER]: Advancing to term 3
I20240502 14:14:23.069891 29062 raft_consensus.cc:504] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:23.070576 29062 leader_election.cc:288] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 3 election: Requested vote from peers d3ff4b86b3e846c9ba04b989f823a000 (127.27.220.130:41255), f14e011f333f4dbeb909c21e11b48ea4 (127.27.220.131:38563)
I20240502 14:14:23.071024 28971 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 3 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "f14e011f333f4dbeb909c21e11b48ea4"
I20240502 14:14:23.071074 28839 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "de95686c7daa435f9627f8dca3fdaa18" candidate_uuid: "c9eb1da45f824ba89668a67f863f6c03" candidate_term: 3 candidate_status { last_received { term: 1 index: 1 } } ignore_live_leader: false dest_uuid: "d3ff4b86b3e846c9ba04b989f823a000"
I20240502 14:14:23.071300 28839 raft_consensus.cc:3037] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 2 FOLLOWER]: Advancing to term 3
I20240502 14:14:23.073141 28839 raft_consensus.cc:2454] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c9eb1da45f824ba89668a67f863f6c03 in term 3.
I20240502 14:14:23.073482 28648 leader_election.cc:302] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: c9eb1da45f824ba89668a67f863f6c03, d3ff4b86b3e846c9ba04b989f823a000; no voters: f14e011f333f4dbeb909c21e11b48ea4
I20240502 14:14:23.073758 29062 raft_consensus.cc:2781] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 3 FOLLOWER]: Leader election won for term 3
I20240502 14:14:23.073880 29062 raft_consensus.cc:686] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [term 3 LEADER]: Becoming Leader. State: Replica: c9eb1da45f824ba89668a67f863f6c03, State: Running, Role: LEADER
I20240502 14:14:23.074117 29062 consensus_queue.cc:237] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 1.1, Last appended by leader: 1, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } }
I20240502 14:14:23.076575 28571 catalog_manager.cc:5484] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 reported cstate change: term changed from 1 to 3. New cstate: current_term: 3 leader_uuid: "c9eb1da45f824ba89668a67f863f6c03" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c9eb1da45f824ba89668a67f863f6c03" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33477 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 } health_report { overall_health: UNKNOWN } } }
W20240502 14:14:23.088433 28934 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:57400: Illegal state: replica f14e011f333f4dbeb909c21e11b48ea4 is not leader of this config: current role FOLLOWER
I20240502 14:14:23.091262 28971 raft_consensus.cc:3037] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 1 FOLLOWER]: Advancing to term 3
I20240502 14:14:23.091414 28839 raft_consensus.cc:1261] T de95686c7daa435f9627f8dca3fdaa18 P d3ff4b86b3e846c9ba04b989f823a000 [term 3 FOLLOWER]: Refusing update from remote peer c9eb1da45f824ba89668a67f863f6c03: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 3 index: 3. (index mismatch)
I20240502 14:14:23.091938 29087 consensus_queue.cc:1035] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d3ff4b86b3e846c9ba04b989f823a000" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 41255 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:23.093900 28971 raft_consensus.cc:1261] T de95686c7daa435f9627f8dca3fdaa18 P f14e011f333f4dbeb909c21e11b48ea4 [term 3 FOLLOWER]: Refusing update from remote peer c9eb1da45f824ba89668a67f863f6c03: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 3 index: 3. (index mismatch)
I20240502 14:14:23.094921 29087 consensus_queue.cc:1035] T de95686c7daa435f9627f8dca3fdaa18 P c9eb1da45f824ba89668a67f863f6c03 [LEADER]: Connected to new peer: Peer: permanent_uuid: "f14e011f333f4dbeb909c21e11b48ea4" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38563 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:23.095726 29209 mvcc.cc:205] Tried to move back new op lower bound from 7023244341617561600 to 7023244333663240192. Current Snapshot: MvccSnapshot[applied={T|T < 7023244341617561600}]
I20240502 14:14:23.096423 29131 mvcc.cc:205] Tried to move back new op lower bound from 7023244341617561600 to 7023244333663240192. Current Snapshot: MvccSnapshot[applied={T|T < 7023244341617561600}]
INSERT report
rows total: 2000
time total: 680.802 ms
time per row: 0.340401 ms
I20240502 14:14:23.705982 28688 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:14:23.710183 28939 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:14:23.718775 28821 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:14:23.874771 28688 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:14:23.875705 28821 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:14:23.875844 28939 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:14:24.050264 28821 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:14:24.054219 28688 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:14:24.056948 28939 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:14:24.675763 28688 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:14:24.675882 28939 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:14:24.677459 28821 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:14:24.834729 28688 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:14:24.837967 28821 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:14:24.840238 28939 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:14:24.977649 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 28632
I20240502 14:14:24.984575 28754 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:25.106577 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 28632
W20240502 14:14:25.127424 28777 connection.cc:535] client connection to 127.27.220.129:33477 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:14:25.127591 28907 connection.cc:535] client connection to 127.27.220.129:33477 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:14:25.127737 28777 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:14:25.127853 28907 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:14:25.127425 28779 connection.cc:535] server connection from 127.27.220.129:45035 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:14:25.128310 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 28762
W20240502 14:14:25.130262 28907 consensus_peers.cc:485] T 222aabe6414f498fa3749f2f83873903 P f14e011f333f4dbeb909c21e11b48ea4 -> Peer c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477): Couldn't send request to peer c9eb1da45f824ba89668a67f863f6c03. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:14:25.130596 28777 consensus_peers.cc:485] T bc33ffa96d0d4b498f44531cb67c342c P d3ff4b86b3e846c9ba04b989f823a000 -> Peer c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477): Couldn't send request to peer c9eb1da45f824ba89668a67f863f6c03. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:14:25.130754 28777 consensus_peers.cc:485] T bb281e01d01f457f84364f0b432a676e P d3ff4b86b3e846c9ba04b989f823a000 -> Peer c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477): Couldn't send request to peer c9eb1da45f824ba89668a67f863f6c03. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:25.135001 28884 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:14:25.179627 28907 consensus_peers.cc:485] T 840e603053f94d7192f26b0dfda184f6 P f14e011f333f4dbeb909c21e11b48ea4 -> Peer c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477): Couldn't send request to peer c9eb1da45f824ba89668a67f863f6c03. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:14:25.182168 28907 consensus_peers.cc:485] T 2bcfe7bce7af41b79ed1415e7a8b76ed P f14e011f333f4dbeb909c21e11b48ea4 -> Peer c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477): Couldn't send request to peer c9eb1da45f824ba89668a67f863f6c03. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:25.254462 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 28762
I20240502 14:14:25.275439 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 28892
W20240502 14:14:25.277192 28907 consensus_peers.cc:485] T f7a111dfadd448bab1d5ffdffd2a9fbc P f14e011f333f4dbeb909c21e11b48ea4 -> Peer c9eb1da45f824ba89668a67f863f6c03 (127.27.220.129:33477): Couldn't send request to peer c9eb1da45f824ba89668a67f863f6c03. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33477: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:25.281081 29014 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:25.402349 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 28892
I20240502 14:14:25.423321 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 28542
I20240502 14:14:25.424458 28603 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:25.544258 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 28542
2024-05-02T14:14:25Z chronyd exiting
[ OK ] TableRangeRebalancingTest.Basic (10438 ms)
[----------] 1 test from TableRangeRebalancingTest (10438 ms total)
[----------] 1 test from RebalanceStartSafetyTest
[ RUN ] RebalanceStartSafetyTest.TooManyIgnoredTservers/1
I20240502 14:14:25.597908 28530 test_util.cc:268] Using random seed: 946852286
I20240502 14:14:25.599653 28530 ts_itest-base.cc:112] Starting cluster with:
I20240502 14:14:25.599711 28530 ts_itest-base.cc:113] --------------
I20240502 14:14:25.599771 28530 ts_itest-base.cc:114] 5 tablet servers
I20240502 14:14:25.599824 28530 ts_itest-base.cc:115] 3 replicas per TS
I20240502 14:14:25.599895 28530 ts_itest-base.cc:116] --------------
2024-05-02T14:14:25Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T14:14:25Z Disabled control of system clock
I20240502 14:14:25.622972 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:42341
--webserver_interface=127.27.220.190
--webserver_port=0
--builtin_ntp_servers=127.27.220.148:44859
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.27.220.190:42341
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 14:14:25.931841 29404 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:25.932160 29404 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:25.932257 29404 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:25.939518 29404 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:25.939642 29404 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 14:14:25.939700 29404 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:25.939762 29404 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 14:14:25.939811 29404 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 14:14:25.948627 29404 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:44859
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.27.220.190:42341
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:42341
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.27.220.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:25.949983 29404 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:25.951712 29404 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:25.962625 29410 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:25.963081 29409 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:25.963860 29412 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:25.964572 29404 server_base.cc:1034] running on GCE node
I20240502 14:14:25.965296 29404 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:25.966696 29404 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:25.967891 29404 hybrid_clock.cc:648] HybridClock initialized: now 1714659265967842 us; error 37 us; skew 500 ppm
I20240502 14:14:25.968222 29404 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:25.970424 29404 webserver.cc:460] Webserver started at http://127.27.220.190:42025/ using document root <none> and password file <none>
I20240502 14:14:25.971107 29404 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:25.971228 29404 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:25.971542 29404 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:25.974169 29404 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "e5b0dfefa6664aaab0562c4d635d9227"
format_stamp: "Formatted at 2024-05-02 14:14:25 on dist-test-slave-j72s"
I20240502 14:14:25.974942 29404 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "e5b0dfefa6664aaab0562c4d635d9227"
format_stamp: "Formatted at 2024-05-02 14:14:25 on dist-test-slave-j72s"
I20240502 14:14:25.980926 29404 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.001s
I20240502 14:14:25.984241 29418 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:25.985385 29404 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
I20240502 14:14:25.985577 29404 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
uuid: "e5b0dfefa6664aaab0562c4d635d9227"
format_stamp: "Formatted at 2024-05-02 14:14:25 on dist-test-slave-j72s"
I20240502 14:14:25.985754 29404 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:26.001899 29404 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:26.002707 29404 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:26.002964 29404 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:26.019615 29404 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.190:42341
I20240502 14:14:26.019621 29469 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.190:42341 every 8 connection(s)
I20240502 14:14:26.020855 29404 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 14:14:26.024313 29470 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:26.029362 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 29404
I20240502 14:14:26.029592 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 14:14:26.033964 29470 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227: Bootstrap starting.
I20240502 14:14:26.037755 29470 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:26.039049 29470 log.cc:824] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:26.041963 29470 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227: No bootstrap required, opened a new log
I20240502 14:14:26.048789 29470 raft_consensus.cc:348] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5b0dfefa6664aaab0562c4d635d9227" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42341 } }
I20240502 14:14:26.049098 29470 raft_consensus.cc:374] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:26.049172 29470 raft_consensus.cc:729] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5b0dfefa6664aaab0562c4d635d9227, State: Initialized, Role: FOLLOWER
I20240502 14:14:26.049703 29470 consensus_queue.cc:260] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5b0dfefa6664aaab0562c4d635d9227" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42341 } }
I20240502 14:14:26.049893 29470 raft_consensus.cc:388] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:14:26.049997 29470 raft_consensus.cc:482] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:14:26.050112 29470 raft_consensus.cc:3037] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:26.052453 29470 raft_consensus.cc:504] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5b0dfefa6664aaab0562c4d635d9227" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42341 } }
I20240502 14:14:26.052886 29470 leader_election.cc:302] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e5b0dfefa6664aaab0562c4d635d9227; no voters:
I20240502 14:14:26.053406 29470 leader_election.cc:288] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:14:26.053573 29475 raft_consensus.cc:2781] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:26.054169 29475 raft_consensus.cc:686] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [term 1 LEADER]: Becoming Leader. State: Replica: e5b0dfefa6664aaab0562c4d635d9227, State: Running, Role: LEADER
I20240502 14:14:26.054679 29475 consensus_queue.cc:237] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5b0dfefa6664aaab0562c4d635d9227" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42341 } }
I20240502 14:14:26.055292 29470 sys_catalog.cc:564] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 14:14:26.057412 29477 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [sys.catalog]: SysCatalogTable state changed. Reason: New leader e5b0dfefa6664aaab0562c4d635d9227. Latest consensus state: current_term: 1 leader_uuid: "e5b0dfefa6664aaab0562c4d635d9227" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5b0dfefa6664aaab0562c4d635d9227" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42341 } } }
I20240502 14:14:26.057392 29476 sys_catalog.cc:455] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "e5b0dfefa6664aaab0562c4d635d9227" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5b0dfefa6664aaab0562c4d635d9227" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42341 } } }
I20240502 14:14:26.057859 29476 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [sys.catalog]: This master's current role is: LEADER
I20240502 14:14:26.059331 29480 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 14:14:26.059497 29477 sys_catalog.cc:458] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227 [sys.catalog]: This master's current role is: LEADER
I20240502 14:14:26.064141 29480 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 14:14:26.072083 29480 catalog_manager.cc:1347] Generated new cluster ID: 18fb599df1fd44eeb60cddb350612640
I20240502 14:14:26.072197 29480 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 14:14:26.102406 29480 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 14:14:26.103618 29480 catalog_manager.cc:1504] Loading token signing keys...
I20240502 14:14:26.117449 29480 catalog_manager.cc:5857] T 00000000000000000000000000000000 P e5b0dfefa6664aaab0562c4d635d9227: Generated new TSK 0
I20240502 14:14:26.118335 29480 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 14:14:26.132846 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.129:0
--local_ip_for_outbound_sockets=127.27.220.129
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--builtin_ntp_servers=127.27.220.148:44859
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:14:26.452662 29494 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:26.452975 29494 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:26.453061 29494 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:26.460532 29494 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:26.460669 29494 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:26.460793 29494 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.129
I20240502 14:14:26.469861 29494 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:44859
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.129
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:26.471174 29494 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:26.472929 29494 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:26.484609 29499 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:26.485344 29500 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:26.486272 29502 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:26.486876 29494 server_base.cc:1034] running on GCE node
I20240502 14:14:26.487565 29494 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:26.488581 29494 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:26.489910 29494 hybrid_clock.cc:648] HybridClock initialized: now 1714659266489806 us; error 34 us; skew 500 ppm
I20240502 14:14:26.490564 29494 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:26.493398 29494 webserver.cc:460] Webserver started at http://127.27.220.129:35353/ using document root <none> and password file <none>
I20240502 14:14:26.494149 29494 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:26.494246 29494 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:26.494518 29494 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:26.497051 29494 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "69527bc88b4d42a49ddd9483fba113c1"
format_stamp: "Formatted at 2024-05-02 14:14:26 on dist-test-slave-j72s"
I20240502 14:14:26.497844 29494 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "69527bc88b4d42a49ddd9483fba113c1"
format_stamp: "Formatted at 2024-05-02 14:14:26 on dist-test-slave-j72s"
I20240502 14:14:26.502774 29494 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240502 14:14:26.506067 29508 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:26.507364 29494 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
I20240502 14:14:26.507510 29494 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "69527bc88b4d42a49ddd9483fba113c1"
format_stamp: "Formatted at 2024-05-02 14:14:26 on dist-test-slave-j72s"
I20240502 14:14:26.507699 29494 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:26.520519 29494 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:26.521268 29494 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:26.521569 29494 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:26.522470 29494 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:26.523882 29494 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:26.523969 29494 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:26.524087 29494 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:26.524169 29494 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:26.554817 29494 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.129:39981
I20240502 14:14:26.554894 29620 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.129:39981 every 8 connection(s)
I20240502 14:14:26.556114 29494 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 14:14:26.560292 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 29494
I20240502 14:14:26.560503 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 14:14:26.563459 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.130:0
--local_ip_for_outbound_sockets=127.27.220.130
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--builtin_ntp_servers=127.27.220.148:44859
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:26.568521 29621 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42341
I20240502 14:14:26.568821 29621 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:26.569777 29621 heartbeater.cc:511] Master 127.27.220.190:42341 requested a full tablet report, sending...
I20240502 14:14:26.571851 29435 ts_manager.cc:194] Registered new tserver with Master: 69527bc88b4d42a49ddd9483fba113c1 (127.27.220.129:39981)
I20240502 14:14:26.573268 29435 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.129:47179
W20240502 14:14:26.873416 29625 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:26.873740 29625 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:26.873879 29625 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:26.880739 29625 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:26.880851 29625 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:26.881004 29625 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.130
I20240502 14:14:26.889509 29625 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:44859
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.130
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:26.890839 29625 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:26.892556 29625 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:26.904608 29631 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:26.905342 29630 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:26.905884 29633 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:26.906836 29625 server_base.cc:1034] running on GCE node
I20240502 14:14:26.907379 29625 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:26.908214 29625 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:26.909411 29625 hybrid_clock.cc:648] HybridClock initialized: now 1714659266909371 us; error 30 us; skew 500 ppm
I20240502 14:14:26.909745 29625 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:26.911888 29625 webserver.cc:460] Webserver started at http://127.27.220.130:37949/ using document root <none> and password file <none>
I20240502 14:14:26.912621 29625 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:26.912737 29625 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:26.913044 29625 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:26.915549 29625 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "6de568d1df684f349865752d8c55876f"
format_stamp: "Formatted at 2024-05-02 14:14:26 on dist-test-slave-j72s"
I20240502 14:14:26.916257 29625 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "6de568d1df684f349865752d8c55876f"
format_stamp: "Formatted at 2024-05-02 14:14:26 on dist-test-slave-j72s"
I20240502 14:14:26.921069 29625 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20240502 14:14:26.924345 29639 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:26.925596 29625 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
I20240502 14:14:26.925756 29625 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "6de568d1df684f349865752d8c55876f"
format_stamp: "Formatted at 2024-05-02 14:14:26 on dist-test-slave-j72s"
I20240502 14:14:26.925956 29625 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:26.938776 29625 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:26.939579 29625 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:26.939841 29625 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:26.940716 29625 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:26.942166 29625 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:26.942273 29625 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:26.942415 29625 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:26.942497 29625 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:26.974191 29625 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.130:40621
I20240502 14:14:26.974254 29751 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.130:40621 every 8 connection(s)
I20240502 14:14:26.975505 29625 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 14:14:26.980950 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 29625
I20240502 14:14:26.981106 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 14:14:26.984011 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:0
--local_ip_for_outbound_sockets=127.27.220.131
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--builtin_ntp_servers=127.27.220.148:44859
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:26.987779 29752 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42341
I20240502 14:14:26.988065 29752 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:26.988924 29752 heartbeater.cc:511] Master 127.27.220.190:42341 requested a full tablet report, sending...
I20240502 14:14:26.990408 29435 ts_manager.cc:194] Registered new tserver with Master: 6de568d1df684f349865752d8c55876f (127.27.220.130:40621)
I20240502 14:14:26.991091 29435 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.130:45687
W20240502 14:14:27.297016 29755 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:27.297281 29755 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:27.297433 29755 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:27.304420 29755 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:27.304529 29755 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:27.304646 29755 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:14:27.313385 29755 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:44859
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:27.314642 29755 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:27.316287 29755 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:27.327572 29761 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:27.328287 29760 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:27.328852 29763 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:27.329675 29755 server_base.cc:1034] running on GCE node
I20240502 14:14:27.330252 29755 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:27.331230 29755 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:27.332437 29755 hybrid_clock.cc:648] HybridClock initialized: now 1714659267332391 us; error 39 us; skew 500 ppm
I20240502 14:14:27.332752 29755 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:27.335003 29755 webserver.cc:460] Webserver started at http://127.27.220.131:39797/ using document root <none> and password file <none>
I20240502 14:14:27.335731 29755 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:27.335861 29755 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:27.336179 29755 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:27.338845 29755 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "fdcd4debe1b741cbb67052f5f3fb78ed"
format_stamp: "Formatted at 2024-05-02 14:14:27 on dist-test-slave-j72s"
I20240502 14:14:27.339573 29755 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "fdcd4debe1b741cbb67052f5f3fb78ed"
format_stamp: "Formatted at 2024-05-02 14:14:27 on dist-test-slave-j72s"
I20240502 14:14:27.344401 29755 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.001s
I20240502 14:14:27.347617 29769 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:27.348872 29755 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:27.349021 29755 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "fdcd4debe1b741cbb67052f5f3fb78ed"
format_stamp: "Formatted at 2024-05-02 14:14:27 on dist-test-slave-j72s"
I20240502 14:14:27.349207 29755 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:27.363086 29755 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:27.363881 29755 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:27.364133 29755 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:27.364995 29755 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:27.366425 29755 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:27.366521 29755 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:27.366627 29755 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:27.366694 29755 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:27.397771 29755 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:42667
I20240502 14:14:27.397835 29881 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:42667 every 8 connection(s)
I20240502 14:14:27.399158 29755 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:14:27.401870 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 29755
I20240502 14:14:27.402070 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 14:14:27.405412 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.132:0
--local_ip_for_outbound_sockets=127.27.220.132
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--builtin_ntp_servers=127.27.220.148:44859
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:27.412164 29882 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42341
I20240502 14:14:27.412473 29882 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:27.413341 29882 heartbeater.cc:511] Master 127.27.220.190:42341 requested a full tablet report, sending...
I20240502 14:14:27.414758 29435 ts_manager.cc:194] Registered new tserver with Master: fdcd4debe1b741cbb67052f5f3fb78ed (127.27.220.131:42667)
I20240502 14:14:27.415505 29435 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:46291
I20240502 14:14:27.576629 29621 heartbeater.cc:503] Master 127.27.220.190:42341 was elected leader, sending a full tablet report...
W20240502 14:14:27.723371 29885 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:27.723676 29885 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:27.723804 29885 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:27.731031 29885 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:27.731179 29885 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:27.731336 29885 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.132
I20240502 14:14:27.740043 29885 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:44859
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.132
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:27.741456 29885 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:27.743176 29885 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:27.755683 29890 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:27.756457 29891 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:27.757040 29893 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:27.757895 29885 server_base.cc:1034] running on GCE node
I20240502 14:14:27.758486 29885 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:27.759441 29885 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:27.760637 29885 hybrid_clock.cc:648] HybridClock initialized: now 1714659267760591 us; error 48 us; skew 500 ppm
I20240502 14:14:27.760939 29885 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:27.768206 29885 webserver.cc:460] Webserver started at http://127.27.220.132:42667/ using document root <none> and password file <none>
I20240502 14:14:27.768971 29885 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:27.769085 29885 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:27.769387 29885 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:27.772076 29885 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b"
format_stamp: "Formatted at 2024-05-02 14:14:27 on dist-test-slave-j72s"
I20240502 14:14:27.772789 29885 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b"
format_stamp: "Formatted at 2024-05-02 14:14:27 on dist-test-slave-j72s"
I20240502 14:14:27.777777 29885 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
I20240502 14:14:27.780903 29899 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:27.782159 29885 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:27.782315 29885 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b"
format_stamp: "Formatted at 2024-05-02 14:14:27 on dist-test-slave-j72s"
I20240502 14:14:27.782496 29885 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:27.798566 29885 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:27.799384 29885 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:27.799633 29885 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:27.800496 29885 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:27.801928 29885 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:27.802019 29885 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:27.802122 29885 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:27.802189 29885 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:27.832459 29885 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.132:38643
I20240502 14:14:27.832481 30011 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.132:38643 every 8 connection(s)
I20240502 14:14:27.833845 29885 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 14:14:27.842927 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 29885
I20240502 14:14:27.843109 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 14:14:27.845700 30012 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42341
I20240502 14:14:27.845978 30012 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:27.846300 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.133:0
--local_ip_for_outbound_sockets=127.27.220.133
--webserver_interface=127.27.220.133
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--builtin_ntp_servers=127.27.220.148:44859
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:27.846752 30012 heartbeater.cc:511] Master 127.27.220.190:42341 requested a full tablet report, sending...
I20240502 14:14:27.848021 29434 ts_manager.cc:194] Registered new tserver with Master: f9ea5fd2f44b4a25b51ddcfc0a88263b (127.27.220.132:38643)
I20240502 14:14:27.848816 29434 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.132:57653
I20240502 14:14:27.993470 29752 heartbeater.cc:503] Master 127.27.220.190:42341 was elected leader, sending a full tablet report...
W20240502 14:14:28.160791 30015 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:28.161069 30015 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:28.161223 30015 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:28.168279 30015 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:28.168396 30015 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:28.168512 30015 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.133
I20240502 14:14:28.177069 30015 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:44859
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.27.220.133
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42341
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.133
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:28.178354 30015 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:28.180030 30015 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:28.192003 30021 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:28.192721 30023 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:28.193238 30020 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:28.194041 30015 server_base.cc:1034] running on GCE node
I20240502 14:14:28.194586 30015 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:28.195425 30015 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:28.196624 30015 hybrid_clock.cc:648] HybridClock initialized: now 1714659268196584 us; error 27 us; skew 500 ppm
I20240502 14:14:28.196915 30015 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:28.199167 30015 webserver.cc:460] Webserver started at http://127.27.220.133:37337/ using document root <none> and password file <none>
I20240502 14:14:28.200037 30015 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:28.200162 30015 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:28.200472 30015 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:28.202975 30015 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "ab1ee15b8b8a4dceacc5e0b4d7a748a5"
format_stamp: "Formatted at 2024-05-02 14:14:28 on dist-test-slave-j72s"
I20240502 14:14:28.203678 30015 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "ab1ee15b8b8a4dceacc5e0b4d7a748a5"
format_stamp: "Formatted at 2024-05-02 14:14:28 on dist-test-slave-j72s"
I20240502 14:14:28.208736 30015 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.004s
I20240502 14:14:28.211949 30029 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:28.213194 30015 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:28.213361 30015 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "ab1ee15b8b8a4dceacc5e0b4d7a748a5"
format_stamp: "Formatted at 2024-05-02 14:14:28 on dist-test-slave-j72s"
I20240502 14:14:28.213579 30015 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:28.236285 30015 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:28.237087 30015 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:28.237360 30015 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:28.238299 30015 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:28.239697 30015 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:28.239799 30015 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:28.239913 30015 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:28.239982 30015 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:28.271335 30015 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.133:36201
I20240502 14:14:28.271389 30141 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.133:36201 every 8 connection(s)
I20240502 14:14:28.272642 30015 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 14:14:28.273188 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 30015
I20240502 14:14:28.273406 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 14:14:28.284970 30142 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42341
I20240502 14:14:28.285269 30142 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:28.286120 30142 heartbeater.cc:511] Master 127.27.220.190:42341 requested a full tablet report, sending...
I20240502 14:14:28.287400 29434 ts_manager.cc:194] Registered new tserver with Master: ab1ee15b8b8a4dceacc5e0b4d7a748a5 (127.27.220.133:36201)
I20240502 14:14:28.288483 29434 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.133:52201
I20240502 14:14:28.289455 28530 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 14:14:28.304329 29434 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55724:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 14:14:28.333318 29687 tablet_service.cc:1467] Processing CreateTablet for tablet 7492d988ceab453985c28ca0dac47b6e (DEFAULT_TABLE table=TestTable [id=fe7d3e874c074a06ad7c120a62158cb3]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:28.334719 29687 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7492d988ceab453985c28ca0dac47b6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:28.334785 29817 tablet_service.cc:1467] Processing CreateTablet for tablet 7492d988ceab453985c28ca0dac47b6e (DEFAULT_TABLE table=TestTable [id=fe7d3e874c074a06ad7c120a62158cb3]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:28.335848 29817 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7492d988ceab453985c28ca0dac47b6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:28.337088 29947 tablet_service.cc:1467] Processing CreateTablet for tablet 7492d988ceab453985c28ca0dac47b6e (DEFAULT_TABLE table=TestTable [id=fe7d3e874c074a06ad7c120a62158cb3]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:28.339079 29947 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7492d988ceab453985c28ca0dac47b6e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:28.345927 30160 tablet_bootstrap.cc:492] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed: Bootstrap starting.
I20240502 14:14:28.348899 30161 tablet_bootstrap.cc:492] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f: Bootstrap starting.
I20240502 14:14:28.349143 30160 tablet_bootstrap.cc:654] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:28.350414 30160 log.cc:824] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:28.352458 30161 tablet_bootstrap.cc:654] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:28.352862 30163 tablet_bootstrap.cc:492] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b: Bootstrap starting.
I20240502 14:14:28.353468 30160 tablet_bootstrap.cc:492] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed: No bootstrap required, opened a new log
I20240502 14:14:28.353863 30160 ts_tablet_manager.cc:1367] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed: Time spent bootstrapping tablet: real 0.008s user 0.005s sys 0.002s
I20240502 14:14:28.354161 30161 log.cc:824] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:28.355953 30163 tablet_bootstrap.cc:654] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:28.356971 30161 tablet_bootstrap.cc:492] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f: No bootstrap required, opened a new log
I20240502 14:14:28.357146 30163 log.cc:824] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:28.357244 30161 ts_tablet_manager.cc:1367] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.003s
I20240502 14:14:28.359818 30163 tablet_bootstrap.cc:492] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b: No bootstrap required, opened a new log
I20240502 14:14:28.360070 30163 ts_tablet_manager.cc:1367] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20240502 14:14:28.360371 30160 raft_consensus.cc:348] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.360836 30160 raft_consensus.cc:374] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:28.360958 30160 raft_consensus.cc:729] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fdcd4debe1b741cbb67052f5f3fb78ed, State: Initialized, Role: FOLLOWER
I20240502 14:14:28.361632 30160 consensus_queue.cc:260] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.362658 29882 heartbeater.cc:503] Master 127.27.220.190:42341 was elected leader, sending a full tablet report...
I20240502 14:14:28.363030 30160 ts_tablet_manager.cc:1398] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed: Time spent starting tablet: real 0.009s user 0.007s sys 0.001s
I20240502 14:14:28.366307 30161 raft_consensus.cc:348] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.366776 30161 raft_consensus.cc:374] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:28.366919 30161 raft_consensus.cc:729] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6de568d1df684f349865752d8c55876f, State: Initialized, Role: FOLLOWER
I20240502 14:14:28.367413 30163 raft_consensus.cc:348] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.367589 30161 consensus_queue.cc:260] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.367805 30163 raft_consensus.cc:374] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:28.367923 30163 raft_consensus.cc:729] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f9ea5fd2f44b4a25b51ddcfc0a88263b, State: Initialized, Role: FOLLOWER
I20240502 14:14:28.368615 30163 consensus_queue.cc:260] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.369436 30161 ts_tablet_manager.cc:1398] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f: Time spent starting tablet: real 0.012s user 0.008s sys 0.004s
I20240502 14:14:28.370375 30012 heartbeater.cc:503] Master 127.27.220.190:42341 was elected leader, sending a full tablet report...
I20240502 14:14:28.370944 30163 ts_tablet_manager.cc:1398] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b: Time spent starting tablet: real 0.011s user 0.009s sys 0.000s
I20240502 14:14:28.388816 30167 raft_consensus.cc:482] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:28.389024 30167 raft_consensus.cc:504] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.390595 30167 leader_election.cc:288] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fdcd4debe1b741cbb67052f5f3fb78ed (127.27.220.131:42667), f9ea5fd2f44b4a25b51ddcfc0a88263b (127.27.220.132:38643)
I20240502 14:14:28.398095 29837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7492d988ceab453985c28ca0dac47b6e" candidate_uuid: "6de568d1df684f349865752d8c55876f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" is_pre_election: true
I20240502 14:14:28.398424 29967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7492d988ceab453985c28ca0dac47b6e" candidate_uuid: "6de568d1df684f349865752d8c55876f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" is_pre_election: true
I20240502 14:14:28.398602 29837 raft_consensus.cc:2454] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6de568d1df684f349865752d8c55876f in term 0.
I20240502 14:14:28.398931 29967 raft_consensus.cc:2454] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6de568d1df684f349865752d8c55876f in term 0.
I20240502 14:14:28.399221 29640 leader_election.cc:302] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6de568d1df684f349865752d8c55876f, fdcd4debe1b741cbb67052f5f3fb78ed; no voters:
I20240502 14:14:28.399655 30167 raft_consensus.cc:2781] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:28.399824 30167 raft_consensus.cc:482] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:28.399926 30167 raft_consensus.cc:3037] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 0 FOLLOWER]: Advancing to term 1
W20240502 14:14:28.401350 29883 tablet.cc:2369] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:14:28.402878 30167 raft_consensus.cc:504] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.403410 30167 leader_election.cc:288] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [CANDIDATE]: Term 1 election: Requested vote from peers fdcd4debe1b741cbb67052f5f3fb78ed (127.27.220.131:42667), f9ea5fd2f44b4a25b51ddcfc0a88263b (127.27.220.132:38643)
I20240502 14:14:28.403950 29837 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7492d988ceab453985c28ca0dac47b6e" candidate_uuid: "6de568d1df684f349865752d8c55876f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed"
I20240502 14:14:28.403993 29967 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7492d988ceab453985c28ca0dac47b6e" candidate_uuid: "6de568d1df684f349865752d8c55876f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b"
I20240502 14:14:28.404163 29967 raft_consensus.cc:3037] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:28.404167 29837 raft_consensus.cc:3037] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:28.406333 29967 raft_consensus.cc:2454] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6de568d1df684f349865752d8c55876f in term 1.
I20240502 14:14:28.406710 29640 leader_election.cc:302] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6de568d1df684f349865752d8c55876f, f9ea5fd2f44b4a25b51ddcfc0a88263b; no voters:
I20240502 14:14:28.406960 30167 raft_consensus.cc:2781] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:28.407418 29837 raft_consensus.cc:2454] T 7492d988ceab453985c28ca0dac47b6e P fdcd4debe1b741cbb67052f5f3fb78ed [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6de568d1df684f349865752d8c55876f in term 1.
I20240502 14:14:28.407438 30167 raft_consensus.cc:686] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [term 1 LEADER]: Becoming Leader. State: Replica: 6de568d1df684f349865752d8c55876f, State: Running, Role: LEADER
W20240502 14:14:28.407783 29878 messenger.cc:188] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.27.220.131:42667
I20240502 14:14:28.407969 30167 consensus_queue.cc:237] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } }
I20240502 14:14:28.412359 29435 catalog_manager.cc:5484] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f reported cstate change: term changed from 0 to 1, leader changed from <none> to 6de568d1df684f349865752d8c55876f (127.27.220.130). New cstate: current_term: 1 leader_uuid: "6de568d1df684f349865752d8c55876f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6de568d1df684f349865752d8c55876f" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 40621 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:28.427245 28530 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 14:14:28.429780 28530 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 69527bc88b4d42a49ddd9483fba113c1 to finish bootstrapping
I20240502 14:14:28.438563 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 6de568d1df684f349865752d8c55876f to finish bootstrapping
I20240502 14:14:28.446254 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver fdcd4debe1b741cbb67052f5f3fb78ed to finish bootstrapping
I20240502 14:14:28.452980 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver f9ea5fd2f44b4a25b51ddcfc0a88263b to finish bootstrapping
I20240502 14:14:28.459859 28530 ts_itest-base.cc:233] Waiting for 0 tablets on tserver ab1ee15b8b8a4dceacc5e0b4d7a748a5 to finish bootstrapping
W20240502 14:14:28.478044 29753 tablet.cc:2369] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 14:14:28.585754 30013 tablet.cc:2369] T 7492d988ceab453985c28ca0dac47b6e P f9ea5fd2f44b4a25b51ddcfc0a88263b: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:14:28.917114 29687 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:14:28.925976 30077 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:28.940709 29556 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:28.940894 29947 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:28.943866 29817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:28.991286 30207 consensus_queue.cc:1035] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [LEADER]: Connected to new peer: Peer: permanent_uuid: "f9ea5fd2f44b4a25b51ddcfc0a88263b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 38643 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:28.995720 30207 consensus_queue.cc:1035] T 7492d988ceab453985c28ca0dac47b6e P 6de568d1df684f349865752d8c55876f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fdcd4debe1b741cbb67052f5f3fb78ed" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 42667 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:29.121506 29556 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.124552 30077 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.125020 29947 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.125399 29687 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:14:29.127395 29817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.291118 30142 heartbeater.cc:503] Master 127.27.220.190:42341 was elected leader, sending a full tablet report...
I20240502 14:14:29.295521 29556 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.298053 29687 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:14:29.300848 30077 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.312125 29947 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.325585 29817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.904652 29556 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.913501 30077 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.914606 29947 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:29.915827 29687 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:14:29.927083 29817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 14:14:30.082175 29881 acceptor_pool.cc:285] unable to collect diagnostics on RPC socket 127.27.220.131:42667: Invalid argument: socket address is ambiguous
I20240502 14:14:30.086481 29556 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:30.091635 29687 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:14:30.097198 29947 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:30.097909 29817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:30.111171 30077 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:30.260149 29556 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:30.262281 29687 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:14:30.269951 29947 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:30.273844 29817 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:30.274247 30077 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:30.432323 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 29494
I20240502 14:14:30.433879 29615 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:30.527704 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 29494
I20240502 14:14:30.545749 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 29625
I20240502 14:14:30.546870 29746 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:30.645824 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 29625
I20240502 14:14:30.665844 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 29755
I20240502 14:14:30.666877 29877 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:30.760596 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 29755
I20240502 14:14:30.778640 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 29885
I20240502 14:14:30.779654 30006 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:30.874217 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 29885
I20240502 14:14:30.891904 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 30015
I20240502 14:14:30.892935 30137 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:30.984361 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 30015
I20240502 14:14:31.002136 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 29404
I20240502 14:14:31.003217 29465 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:31.116276 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 29404
2024-05-02T14:14:31Z chronyd exiting
[ OK ] RebalanceStartSafetyTest.TooManyIgnoredTservers/1 (5560 ms)
[----------] 1 test from RebalanceStartSafetyTest (5561 ms total)
[----------] 1 test from RebalanceParamTest
[ RUN ] RebalanceParamTest.Rebalance/7
I20240502 14:14:31.159055 28530 test_util.cc:268] Using random seed: 952413413
I20240502 14:14:31.160921 28530 ts_itest-base.cc:112] Starting cluster with:
I20240502 14:14:31.161015 28530 ts_itest-base.cc:113] --------------
I20240502 14:14:31.161069 28530 ts_itest-base.cc:114] 7 tablet servers
I20240502 14:14:31.161157 28530 ts_itest-base.cc:115] 5 replicas per TS
I20240502 14:14:31.161229 28530 ts_itest-base.cc:116] --------------
2024-05-02T14:14:31Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T14:14:31Z Disabled control of system clock
I20240502 14:14:31.184729 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:33157
--webserver_interface=127.27.220.190
--webserver_port=0
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.27.220.190:33157
--allow_unsafe_replication_factor
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000 with env {}
W20240502 14:14:31.495997 30402 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 14:14:31.496353 30402 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:31.496455 30402 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:31.496557 30402 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:31.503556 30402 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:31.503677 30402 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 14:14:31.503772 30402 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:31.503857 30402 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 14:14:31.503949 30402 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 14:14:31.512296 30402 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.27.220.190:33157
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:33157
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.27.220.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:31.513645 30402 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:31.515309 30402 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:31.525842 30407 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:31.526432 30408 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:31.527117 30410 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:31.527865 30402 server_base.cc:1034] running on GCE node
I20240502 14:14:31.528551 30402 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:31.529906 30402 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:31.531105 30402 hybrid_clock.cc:648] HybridClock initialized: now 1714659271531052 us; error 29 us; skew 500 ppm
I20240502 14:14:31.531404 30402 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:31.533551 30402 webserver.cc:460] Webserver started at http://127.27.220.190:35943/ using document root <none> and password file <none>
I20240502 14:14:31.534174 30402 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:31.534297 30402 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:31.534559 30402 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:31.536796 30402 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "1cb98ed728ea452ab7c2d0de3cb89c33"
format_stamp: "Formatted at 2024-05-02 14:14:31 on dist-test-slave-j72s"
I20240502 14:14:31.537431 30402 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "1cb98ed728ea452ab7c2d0de3cb89c33"
format_stamp: "Formatted at 2024-05-02 14:14:31 on dist-test-slave-j72s"
I20240502 14:14:31.542114 30402 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
I20240502 14:14:31.545133 30416 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:31.546275 30402 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:31.546490 30402 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
uuid: "1cb98ed728ea452ab7c2d0de3cb89c33"
format_stamp: "Formatted at 2024-05-02 14:14:31 on dist-test-slave-j72s"
I20240502 14:14:31.546680 30402 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:31.559641 30402 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:31.560397 30402 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:31.560655 30402 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:31.577335 30402 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.190:33157
I20240502 14:14:31.577358 30467 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.190:33157 every 8 connection(s)
I20240502 14:14:31.578696 30402 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 14:14:31.580487 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 30402
I20240502 14:14:31.580677 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 14:14:31.582571 30468 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:31.595296 30468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33: Bootstrap starting.
I20240502 14:14:31.598299 30468 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:31.599263 30468 log.cc:824] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:31.601462 30468 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33: No bootstrap required, opened a new log
I20240502 14:14:31.606771 30468 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1cb98ed728ea452ab7c2d0de3cb89c33" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 33157 } }
I20240502 14:14:31.607054 30468 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:31.607161 30468 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1cb98ed728ea452ab7c2d0de3cb89c33, State: Initialized, Role: FOLLOWER
I20240502 14:14:31.607718 30468 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1cb98ed728ea452ab7c2d0de3cb89c33" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 33157 } }
I20240502 14:14:31.607909 30468 raft_consensus.cc:388] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:14:31.608044 30468 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:14:31.608184 30468 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:31.610116 30468 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1cb98ed728ea452ab7c2d0de3cb89c33" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 33157 } }
I20240502 14:14:31.610553 30468 leader_election.cc:302] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 1cb98ed728ea452ab7c2d0de3cb89c33; no voters:
I20240502 14:14:31.611060 30468 leader_election.cc:288] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:14:31.611177 30473 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:31.611725 30473 raft_consensus.cc:686] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [term 1 LEADER]: Becoming Leader. State: Replica: 1cb98ed728ea452ab7c2d0de3cb89c33, State: Running, Role: LEADER
I20240502 14:14:31.612289 30473 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1cb98ed728ea452ab7c2d0de3cb89c33" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 33157 } }
I20240502 14:14:31.613236 30468 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 14:14:31.614892 30474 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1cb98ed728ea452ab7c2d0de3cb89c33. Latest consensus state: current_term: 1 leader_uuid: "1cb98ed728ea452ab7c2d0de3cb89c33" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1cb98ed728ea452ab7c2d0de3cb89c33" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 33157 } } }
I20240502 14:14:31.614866 30475 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "1cb98ed728ea452ab7c2d0de3cb89c33" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1cb98ed728ea452ab7c2d0de3cb89c33" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 33157 } } }
I20240502 14:14:31.615294 30474 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [sys.catalog]: This master's current role is: LEADER
I20240502 14:14:31.615304 30475 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33 [sys.catalog]: This master's current role is: LEADER
I20240502 14:14:31.620901 30479 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 14:14:31.625470 30479 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 14:14:31.633277 30479 catalog_manager.cc:1347] Generated new cluster ID: 4d1ed6fee3d943a6a2531b12ce316ce9
I20240502 14:14:31.633368 30479 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 14:14:31.649180 30479 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 14:14:31.650197 30479 catalog_manager.cc:1504] Loading token signing keys...
I20240502 14:14:31.657163 30479 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33: Generated new TSK 0
I20240502 14:14:31.657881 30479 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 14:14:31.665374 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.129:0
--local_ip_for_outbound_sockets=127.27.220.129
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:14:31.978766 30492 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:31.979175 30492 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:31.979329 30492 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:31.987082 30492 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:31.987217 30492 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:31.987370 30492 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.129
I20240502 14:14:31.996058 30492 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.129
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:31.997368 30492 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:31.999135 30492 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:32.011107 30497 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:32.011940 30498 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:32.012555 30500 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:32.013311 30492 server_base.cc:1034] running on GCE node
I20240502 14:14:32.013963 30492 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:32.014883 30492 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:32.016077 30492 hybrid_clock.cc:648] HybridClock initialized: now 1714659272016023 us; error 45 us; skew 500 ppm
I20240502 14:14:32.016387 30492 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:32.018610 30492 webserver.cc:460] Webserver started at http://127.27.220.129:37153/ using document root <none> and password file <none>
I20240502 14:14:32.019340 30492 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:32.019467 30492 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:32.019778 30492 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:32.022640 30492 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "fd3399b1bfe644159a833ede6726c25f"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.023397 30492 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "fd3399b1bfe644159a833ede6726c25f"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.028439 30492 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240502 14:14:32.031896 30506 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.033285 30492 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.001s
I20240502 14:14:32.033444 30492 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "fd3399b1bfe644159a833ede6726c25f"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.033679 30492 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:32.052340 30492 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:32.053221 30492 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:32.053489 30492 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:32.054484 30492 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:32.055910 30492 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:32.056007 30492 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.056124 30492 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:32.056203 30492 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.087322 30492 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.129:33055
I20240502 14:14:32.087374 30618 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.129:33055 every 8 connection(s)
I20240502 14:14:32.088635 30492 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 14:14:32.093899 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 30492
I20240502 14:14:32.094094 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 14:14:32.097659 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.130:0
--local_ip_for_outbound_sockets=127.27.220.130
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:32.102409 30619 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:14:32.102720 30619 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:32.103487 30619 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:14:32.105392 30433 ts_manager.cc:194] Registered new tserver with Master: fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:14:32.106797 30433 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.129:58407
W20240502 14:14:32.401567 30622 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:32.401875 30622 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:32.402020 30622 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:32.409276 30622 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:32.409422 30622 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:32.409615 30622 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.130
I20240502 14:14:32.418457 30622 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.130
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:32.419761 30622 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:32.421489 30622 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:32.433737 30627 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:32.434283 30630 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:32.434927 30628 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:32.435730 30622 server_base.cc:1034] running on GCE node
I20240502 14:14:32.436321 30622 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:32.437279 30622 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:32.438509 30622 hybrid_clock.cc:648] HybridClock initialized: now 1714659272438440 us; error 53 us; skew 500 ppm
I20240502 14:14:32.438854 30622 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:32.441229 30622 webserver.cc:460] Webserver started at http://127.27.220.130:38669/ using document root <none> and password file <none>
I20240502 14:14:32.442032 30622 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:32.442162 30622 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:32.442472 30622 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:32.445091 30622 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "d733c51902044956a8cd2b6b31bc9831"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.445899 30622 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "d733c51902044956a8cd2b6b31bc9831"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.451135 30622 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.002s
I20240502 14:14:32.454530 30636 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.455890 30622 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 14:14:32.456044 30622 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "d733c51902044956a8cd2b6b31bc9831"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.456241 30622 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:32.488600 30622 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:32.489485 30622 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:32.489807 30622 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:32.490782 30622 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:32.492264 30622 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:32.492363 30622 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.492486 30622 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:32.492561 30622 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.524823 30622 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.130:46259
I20240502 14:14:32.524875 30748 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.130:46259 every 8 connection(s)
I20240502 14:14:32.526214 30622 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 14:14:32.536463 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 30622
I20240502 14:14:32.536684 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 14:14:32.538898 30749 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:14:32.539193 30749 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:32.540234 30749 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:14:32.540807 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:0
--local_ip_for_outbound_sockets=127.27.220.131
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:32.541810 30432 ts_manager.cc:194] Registered new tserver with Master: d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259)
I20240502 14:14:32.542601 30432 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.130:51047
W20240502 14:14:32.852722 30752 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:32.853024 30752 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:32.853164 30752 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:32.860718 30752 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:32.860867 30752 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:32.861032 30752 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:14:32.869977 30752 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:32.871315 30752 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:32.873059 30752 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:32.884689 30758 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:32.885460 30757 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:32.886068 30760 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:32.886787 30752 server_base.cc:1034] running on GCE node
I20240502 14:14:32.887336 30752 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:32.888250 30752 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:32.889467 30752 hybrid_clock.cc:648] HybridClock initialized: now 1714659272889420 us; error 38 us; skew 500 ppm
I20240502 14:14:32.889825 30752 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:32.896423 30752 webserver.cc:460] Webserver started at http://127.27.220.131:41385/ using document root <none> and password file <none>
I20240502 14:14:32.897243 30752 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:32.897367 30752 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:32.897778 30752 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:32.900571 30752 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "155f26c73dc84441a58417c09f577494"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.901422 30752 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "155f26c73dc84441a58417c09f577494"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.907176 30752 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240502 14:14:32.910866 30766 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.912259 30752 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 14:14:32.912436 30752 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "155f26c73dc84441a58417c09f577494"
format_stamp: "Formatted at 2024-05-02 14:14:32 on dist-test-slave-j72s"
I20240502 14:14:32.912662 30752 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:32.946249 30752 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:32.947228 30752 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:32.947520 30752 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:32.948740 30752 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:32.950378 30752 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:32.950482 30752 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.950603 30752 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:32.950676 30752 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:32.983625 30752 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:34975
I20240502 14:14:32.983680 30878 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:34975 every 8 connection(s)
I20240502 14:14:32.984951 30752 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:14:32.988960 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 30752
I20240502 14:14:32.989137 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 14:14:32.992666 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.132:0
--local_ip_for_outbound_sockets=127.27.220.132
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:32.997995 30879 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:14:32.998306 30879 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:32.999115 30879 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:14:33.000592 30432 ts_manager.cc:194] Registered new tserver with Master: 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:33.001369 30432 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:44927
I20240502 14:14:33.109493 30619 heartbeater.cc:503] Master 127.27.220.190:33157 was elected leader, sending a full tablet report...
W20240502 14:14:33.310735 30882 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:33.311041 30882 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:33.311172 30882 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:33.318499 30882 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:33.318639 30882 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:33.318789 30882 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.132
I20240502 14:14:33.327656 30882 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.132
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:33.328994 30882 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:33.330783 30882 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:33.343261 30888 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:33.344008 30887 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:33.344588 30890 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:33.345404 30882 server_base.cc:1034] running on GCE node
I20240502 14:14:33.345948 30882 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:33.346849 30882 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:33.348043 30882 hybrid_clock.cc:648] HybridClock initialized: now 1714659273347995 us; error 41 us; skew 500 ppm
I20240502 14:14:33.348351 30882 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:33.350553 30882 webserver.cc:460] Webserver started at http://127.27.220.132:38813/ using document root <none> and password file <none>
I20240502 14:14:33.351312 30882 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:33.351440 30882 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:33.351765 30882 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:33.354364 30882 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
format_stamp: "Formatted at 2024-05-02 14:14:33 on dist-test-slave-j72s"
I20240502 14:14:33.355041 30882 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
format_stamp: "Formatted at 2024-05-02 14:14:33 on dist-test-slave-j72s"
I20240502 14:14:33.360110 30882 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.003s
I20240502 14:14:33.363395 30896 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:33.364677 30882 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:33.364821 30882 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
format_stamp: "Formatted at 2024-05-02 14:14:33 on dist-test-slave-j72s"
I20240502 14:14:33.365010 30882 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:33.384399 30882 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:33.385243 30882 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:33.385505 30882 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:33.386472 30882 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:33.387981 30882 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:33.388089 30882 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:33.388204 30882 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:33.388283 30882 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:33.419368 30882 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.132:41511
I20240502 14:14:33.419417 31008 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.132:41511 every 8 connection(s)
I20240502 14:14:33.420727 30882 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 14:14:33.421213 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 30882
I20240502 14:14:33.421407 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 14:14:33.425300 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.133:0
--local_ip_for_outbound_sockets=127.27.220.133
--webserver_interface=127.27.220.133
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:33.433904 31009 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:14:33.434207 31009 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:33.435040 31009 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:14:33.436444 30432 ts_manager.cc:194] Registered new tserver with Master: 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:14:33.437216 30432 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.132:46161
I20240502 14:14:33.545123 30749 heartbeater.cc:503] Master 127.27.220.190:33157 was elected leader, sending a full tablet report...
W20240502 14:14:33.732836 31012 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:33.733155 31012 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:33.733249 31012 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:33.741040 31012 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:33.741180 31012 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:33.741325 31012 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.133
I20240502 14:14:33.750209 31012 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.27.220.133
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.133
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:33.751642 31012 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:33.753382 31012 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:33.765193 31018 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:33.765743 31017 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:33.766474 31020 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:33.767264 31012 server_base.cc:1034] running on GCE node
I20240502 14:14:33.767818 31012 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:33.768740 31012 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:33.769935 31012 hybrid_clock.cc:648] HybridClock initialized: now 1714659273769890 us; error 41 us; skew 500 ppm
I20240502 14:14:33.770268 31012 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:33.772408 31012 webserver.cc:460] Webserver started at http://127.27.220.133:39897/ using document root <none> and password file <none>
I20240502 14:14:33.773121 31012 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:33.773243 31012 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:33.773571 31012 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:33.776010 31012 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "ba420f2442d6416bbda95edfff85e849"
format_stamp: "Formatted at 2024-05-02 14:14:33 on dist-test-slave-j72s"
I20240502 14:14:33.776726 31012 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "ba420f2442d6416bbda95edfff85e849"
format_stamp: "Formatted at 2024-05-02 14:14:33 on dist-test-slave-j72s"
I20240502 14:14:33.781795 31012 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240502 14:14:33.784991 31026 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:33.786245 31012 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:33.786391 31012 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "ba420f2442d6416bbda95edfff85e849"
format_stamp: "Formatted at 2024-05-02 14:14:33 on dist-test-slave-j72s"
I20240502 14:14:33.786572 31012 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:33.810360 31012 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:33.811151 31012 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:33.811422 31012 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:33.812371 31012 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:33.813899 31012 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:33.813992 31012 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:33.814103 31012 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:33.814183 31012 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:33.845552 31012 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.133:34805
I20240502 14:14:33.845629 31138 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.133:34805 every 8 connection(s)
I20240502 14:14:33.846822 31012 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 14:14:33.852941 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 31012
I20240502 14:14:33.853089 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 14:14:33.855923 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.134:0
--local_ip_for_outbound_sockets=127.27.220.134
--webserver_interface=127.27.220.134
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:33.859354 31139 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:14:33.859622 31139 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:33.860366 31139 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:14:33.861641 30432 ts_manager.cc:194] Registered new tserver with Master: ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:33.862732 30432 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.133:39523
I20240502 14:14:34.004016 30879 heartbeater.cc:503] Master 127.27.220.190:33157 was elected leader, sending a full tablet report...
W20240502 14:14:34.183771 31142 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:34.184052 31142 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:34.184146 31142 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:34.191217 31142 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:34.191331 31142 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:34.191491 31142 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.134
I20240502 14:14:34.200012 31142 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.134:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.27.220.134
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.134
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:34.201314 31142 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:34.202973 31142 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:34.214321 31150 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:34.215034 31147 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:34.215600 31148 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:34.216398 31142 server_base.cc:1034] running on GCE node
I20240502 14:14:34.216915 31142 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:34.217773 31142 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:34.218971 31142 hybrid_clock.cc:648] HybridClock initialized: now 1714659274218924 us; error 38 us; skew 500 ppm
I20240502 14:14:34.219290 31142 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:34.221232 31142 webserver.cc:460] Webserver started at http://127.27.220.134:40611/ using document root <none> and password file <none>
I20240502 14:14:34.221946 31142 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:34.222072 31142 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:34.222328 31142 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:34.224529 31142 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
format_stamp: "Formatted at 2024-05-02 14:14:34 on dist-test-slave-j72s"
I20240502 14:14:34.225143 31142 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
format_stamp: "Formatted at 2024-05-02 14:14:34 on dist-test-slave-j72s"
I20240502 14:14:34.229728 31142 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.002s
I20240502 14:14:34.232823 31156 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:34.234042 31142 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:34.234184 31142 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
format_stamp: "Formatted at 2024-05-02 14:14:34 on dist-test-slave-j72s"
I20240502 14:14:34.234369 31142 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:34.254037 31142 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:34.254801 31142 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:34.255054 31142 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:34.255944 31142 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:34.257385 31142 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:34.257473 31142 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:34.257608 31142 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:34.257675 31142 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:34.288304 31142 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.134:39403
I20240502 14:14:34.288348 31268 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.134:39403 every 8 connection(s)
I20240502 14:14:34.289605 31142 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 14:14:34.297756 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 31142
I20240502 14:14:34.297999 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 14:14:34.301789 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.135:0
--local_ip_for_outbound_sockets=127.27.220.135
--webserver_interface=127.27.220.135
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:14:34.302188 31269 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:14:34.302515 31269 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:34.303555 31269 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:14:34.304972 30432 ts_manager.cc:194] Registered new tserver with Master: 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:34.305780 30432 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.134:34001
I20240502 14:14:34.439703 31009 heartbeater.cc:503] Master 127.27.220.190:33157 was elected leader, sending a full tablet report...
W20240502 14:14:34.611928 31272 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:34.612193 31272 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:34.612318 31272 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:34.619163 31272 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:34.619277 31272 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:34.619414 31272 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.135
I20240502 14:14:34.627453 31272 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.135:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.27.220.135
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.135
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:34.628824 31272 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:34.630450 31272 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:34.641955 31277 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:34.643343 31280 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:34.644001 31278 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:34.644860 31272 server_base.cc:1034] running on GCE node
I20240502 14:14:34.645426 31272 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:34.646344 31272 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:34.647529 31272 hybrid_clock.cc:648] HybridClock initialized: now 1714659274647478 us; error 45 us; skew 500 ppm
I20240502 14:14:34.647835 31272 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:34.650079 31272 webserver.cc:460] Webserver started at http://127.27.220.135:37645/ using document root <none> and password file <none>
I20240502 14:14:34.650804 31272 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:34.650904 31272 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:34.651264 31272 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:14:34.653842 31272 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "3eafb09239024de3bc0eea8d700f7668"
format_stamp: "Formatted at 2024-05-02 14:14:34 on dist-test-slave-j72s"
I20240502 14:14:34.654551 31272 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "3eafb09239024de3bc0eea8d700f7668"
format_stamp: "Formatted at 2024-05-02 14:14:34 on dist-test-slave-j72s"
I20240502 14:14:34.659255 31272 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20240502 14:14:34.662364 31286 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:34.663625 31272 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:34.663772 31272 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "3eafb09239024de3bc0eea8d700f7668"
format_stamp: "Formatted at 2024-05-02 14:14:34 on dist-test-slave-j72s"
I20240502 14:14:34.663951 31272 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:34.678241 31272 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:34.678994 31272 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:34.679260 31272 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:34.679981 31272 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:34.681277 31272 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:34.681355 31272 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:34.681427 31272 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:34.681475 31272 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:34.711498 31272 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.135:33091
I20240502 14:14:34.711544 31398 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.135:33091 every 8 connection(s)
I20240502 14:14:34.712841 31272 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 14:14:34.719352 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 31272
I20240502 14:14:34.719504 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 14:14:34.724999 31399 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:14:34.725266 31399 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:34.726001 31399 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:14:34.727103 30432 ts_manager.cc:194] Registered new tserver with Master: 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091)
I20240502 14:14:34.727917 30432 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.135:51563
I20240502 14:14:34.735060 28530 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240502 14:14:34.750159 30432 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36920:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 14:14:34.796048 30554 tablet_service.cc:1467] Processing CreateTablet for tablet b63847bdc0e546bca632a85255a45572 (DEFAULT_TABLE table=TestTable [id=9f6d350f0a5941b88f4dc33c6d359c08]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:34.796835 30814 tablet_service.cc:1467] Processing CreateTablet for tablet b63847bdc0e546bca632a85255a45572 (DEFAULT_TABLE table=TestTable [id=9f6d350f0a5941b88f4dc33c6d359c08]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:34.797889 30554 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b63847bdc0e546bca632a85255a45572. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:34.798259 30814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b63847bdc0e546bca632a85255a45572. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:34.805611 31074 tablet_service.cc:1467] Processing CreateTablet for tablet b63847bdc0e546bca632a85255a45572 (DEFAULT_TABLE table=TestTable [id=9f6d350f0a5941b88f4dc33c6d359c08]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:34.805922 31204 tablet_service.cc:1467] Processing CreateTablet for tablet b63847bdc0e546bca632a85255a45572 (DEFAULT_TABLE table=TestTable [id=9f6d350f0a5941b88f4dc33c6d359c08]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:34.806113 30944 tablet_service.cc:1467] Processing CreateTablet for tablet b63847bdc0e546bca632a85255a45572 (DEFAULT_TABLE table=TestTable [id=9f6d350f0a5941b88f4dc33c6d359c08]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:34.806952 31074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b63847bdc0e546bca632a85255a45572. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:34.806972 31204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b63847bdc0e546bca632a85255a45572. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:34.807325 30944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b63847bdc0e546bca632a85255a45572. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:34.816030 31421 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:34.821031 31421 tablet_bootstrap.cc:654] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:34.822705 31421 log.cc:824] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:34.829170 31421 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:34.829685 31421 ts_tablet_manager.cc:1367] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.014s user 0.011s sys 0.000s
I20240502 14:14:34.838474 31423 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:34.841629 31422 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:34.842415 31423 tablet_bootstrap.cc:654] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:34.843632 31423 log.cc:824] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:34.844905 31421 raft_consensus.cc:348] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.845418 31421 raft_consensus.cc:374] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:34.845525 31421 raft_consensus.cc:729] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:34.846175 31421 consensus_queue.cc:260] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.846843 31422 tablet_bootstrap.cc:654] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:34.847088 31425 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:34.848127 31422 log.cc:824] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:34.850591 31426 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:34.855144 31426 tablet_bootstrap.cc:654] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:34.856413 31426 log.cc:824] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:34.860497 31425 tablet_bootstrap.cc:654] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:34.861994 31425 log.cc:824] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:34.865387 31139 heartbeater.cc:503] Master 127.27.220.190:33157 was elected leader, sending a full tablet report...
I20240502 14:14:34.865643 31426 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:34.865904 31426 ts_tablet_manager.cc:1367] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.016s user 0.009s sys 0.000s
I20240502 14:14:34.872043 31423 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:34.872462 31423 ts_tablet_manager.cc:1367] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.035s user 0.005s sys 0.009s
I20240502 14:14:34.882234 31425 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:34.882503 31425 ts_tablet_manager.cc:1367] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.036s user 0.022s sys 0.000s
I20240502 14:14:34.882998 31421 ts_tablet_manager.cc:1398] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.053s user 0.018s sys 0.029s
I20240502 14:14:34.884431 31426 raft_consensus.cc:348] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.884836 31426 raft_consensus.cc:374] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:34.884935 31426 raft_consensus.cc:729] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:34.885689 31426 consensus_queue.cc:260] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.886615 31423 raft_consensus.cc:348] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.888604 31422 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:34.888851 31422 ts_tablet_manager.cc:1367] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.048s user 0.017s sys 0.004s
I20240502 14:14:34.889631 31423 raft_consensus.cc:374] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:34.889794 31423 raft_consensus.cc:729] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:34.890539 31423 consensus_queue.cc:260] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.893231 31425 raft_consensus.cc:348] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.893775 31425 raft_consensus.cc:374] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:34.893899 31425 raft_consensus.cc:729] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:34.894804 31425 consensus_queue.cc:260] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.896618 31426 ts_tablet_manager.cc:1398] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.031s user 0.014s sys 0.008s
I20240502 14:14:34.899444 31422 raft_consensus.cc:348] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.899852 31425 ts_tablet_manager.cc:1398] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.017s user 0.012s sys 0.003s
I20240502 14:14:34.899968 31422 raft_consensus.cc:374] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:34.900101 31422 raft_consensus.cc:729] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:34.900883 31422 consensus_queue.cc:260] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:34.901434 31423 ts_tablet_manager.cc:1398] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.029s user 0.020s sys 0.002s
I20240502 14:14:34.902441 31269 heartbeater.cc:503] Master 127.27.220.190:33157 was elected leader, sending a full tablet report...
I20240502 14:14:34.903251 31422 ts_tablet_manager.cc:1398] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.014s user 0.009s sys 0.006s
W20240502 14:14:34.923324 31010 tablet.cc:2369] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 14:14:34.988015 30880 tablet.cc:2369] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 14:14:35.041669 31270 tablet.cc:2369] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 14:14:35.092754 30620 tablet.cc:2369] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 14:14:35.099097 31140 tablet.cc:2369] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:14:35.202951 31428 raft_consensus.cc:482] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:35.203281 31428 raft_consensus.cc:504] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:35.205277 31428 leader_election.cc:288] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:35.215974 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:35.216488 30834 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 0.
I20240502 14:14:35.216775 31224 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:35.217060 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:35.217314 31224 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 0.
I20240502 14:14:35.217581 30964 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 0.
I20240502 14:14:35.217501 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:35.217974 31094 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 0.
I20240502 14:14:35.218183 30507 leader_election.cc:302] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:35.218590 31428 raft_consensus.cc:2781] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:35.218708 31428 raft_consensus.cc:482] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:35.218778 31428 raft_consensus.cc:3037] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:35.220772 31428 raft_consensus.cc:504] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:35.221560 31428 leader_election.cc:288] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [CANDIDATE]: Term 1 election: Requested vote from peers 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:35.221807 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:35.221947 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:35.221987 30834 raft_consensus.cc:3037] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:35.221982 31224 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
I20240502 14:14:35.222121 30964 raft_consensus.cc:3037] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:35.222182 31224 raft_consensus.cc:3037] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:35.222262 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:35.222450 31094 raft_consensus.cc:3037] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:35.224962 30964 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 1.
I20240502 14:14:35.225329 30834 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 1.
I20240502 14:14:35.225351 31094 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 1.
I20240502 14:14:35.225350 31224 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 1.
I20240502 14:14:35.225829 30507 leader_election.cc:302] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 5 voters: 4 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:35.226208 31428 raft_consensus.cc:2781] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:35.226593 31428 raft_consensus.cc:686] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Becoming Leader. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Running, Role: LEADER
I20240502 14:14:35.227068 31428 consensus_queue.cc:237] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } }
I20240502 14:14:35.231451 30431 catalog_manager.cc:5484] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f reported cstate change: term changed from 0 to 1, leader changed from <none> to fd3399b1bfe644159a833ede6726c25f (127.27.220.129). New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:35.264420 28530 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240502 14:14:35.266391 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver fd3399b1bfe644159a833ede6726c25f to finish bootstrapping
I20240502 14:14:35.273907 28530 ts_itest-base.cc:233] Waiting for 0 tablets on tserver d733c51902044956a8cd2b6b31bc9831 to finish bootstrapping
I20240502 14:14:35.281836 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 155f26c73dc84441a58417c09f577494 to finish bootstrapping
I20240502 14:14:35.288251 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 7c1f568f8efc4b2a9dbf38907f02f3e3 to finish bootstrapping
I20240502 14:14:35.296046 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver ba420f2442d6416bbda95edfff85e849 to finish bootstrapping
I20240502 14:14:35.303812 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 0dfd48d0b03a42f689e58d5190d03a2f to finish bootstrapping
I20240502 14:14:35.310328 28530 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 3eafb09239024de3bc0eea8d700f7668 to finish bootstrapping
I20240502 14:14:35.318253 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 31272
I20240502 14:14:35.319092 31394 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:14:35.397118 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 31272
I20240502 14:14:35.606616 31428 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:35.650483 31428 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:35.658310 31452 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:35.658857 31452 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.164901 30433 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36920:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:14:39.176051 30944 tablet_service.cc:1467] Processing CreateTablet for tablet 74c46ec356414d26987e1cd6a0857602 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.176671 30944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 74c46ec356414d26987e1cd6a0857602. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.177821 31204 tablet_service.cc:1467] Processing CreateTablet for tablet 5c4ceedcb00a49e1a32977e4e1acea24 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.177881 30553 tablet_service.cc:1467] Processing CreateTablet for tablet 5c4ceedcb00a49e1a32977e4e1acea24 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.178303 30553 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c4ceedcb00a49e1a32977e4e1acea24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.178296 31204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c4ceedcb00a49e1a32977e4e1acea24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.179693 30554 tablet_service.cc:1467] Processing CreateTablet for tablet 74c46ec356414d26987e1cd6a0857602 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.180240 30554 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 74c46ec356414d26987e1cd6a0857602. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.182552 31074 tablet_service.cc:1467] Processing CreateTablet for tablet 74c46ec356414d26987e1cd6a0857602 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.182983 31074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 74c46ec356414d26987e1cd6a0857602. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.186314 30943 tablet_service.cc:1467] Processing CreateTablet for tablet 5c4ceedcb00a49e1a32977e4e1acea24 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.186703 30943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c4ceedcb00a49e1a32977e4e1acea24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.192584 31073 tablet_service.cc:1467] Processing CreateTablet for tablet 975ae0ae782a45aebda66d2e32713def (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.193150 31073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 975ae0ae782a45aebda66d2e32713def. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.196146 30554 tablet_service.cc:1467] Processing CreateTablet for tablet 975ae0ae782a45aebda66d2e32713def (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.196523 30554 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 975ae0ae782a45aebda66d2e32713def. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.198359 31203 tablet_service.cc:1467] Processing CreateTablet for tablet 975ae0ae782a45aebda66d2e32713def (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.198813 31203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 975ae0ae782a45aebda66d2e32713def. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.200579 31466 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:39.203611 31468 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:39.206072 31468 tablet_bootstrap.cc:654] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.206633 31469 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:39.207935 31469 tablet_bootstrap.cc:654] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.208976 31466 tablet_bootstrap.cc:654] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.211642 31466 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:39.211774 31466 ts_tablet_manager.cc:1367] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.011s user 0.001s sys 0.002s
I20240502 14:14:39.212517 31466 raft_consensus.cc:348] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.212702 31466 raft_consensus.cc:374] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.212787 31466 raft_consensus.cc:729] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.213413 31469 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:39.213591 31469 ts_tablet_manager.cc:1367] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.000s
I20240502 14:14:39.214555 31468 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:39.214751 31468 ts_tablet_manager.cc:1367] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.004s
I20240502 14:14:39.215035 31469 raft_consensus.cc:348] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.215345 31469 raft_consensus.cc:374] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.215494 31469 raft_consensus.cc:729] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.215695 31466 consensus_queue.cc:260] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.215798 31469 consensus_queue.cc:260] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.216295 31466 ts_tablet_manager.cc:1398] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240502 14:14:39.217759 31466 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:39.217310 31468 raft_consensus.cc:348] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.219470 31468 raft_consensus.cc:374] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.219564 31468 raft_consensus.cc:729] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.219789 31468 consensus_queue.cc:260] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.221671 30814 tablet_service.cc:1467] Processing CreateTablet for tablet 74c46ec356414d26987e1cd6a0857602 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.222147 30814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 74c46ec356414d26987e1cd6a0857602. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.223057 30813 tablet_service.cc:1467] Processing CreateTablet for tablet 5c4ceedcb00a49e1a32977e4e1acea24 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.223660 30812 tablet_service.cc:1467] Processing CreateTablet for tablet 975ae0ae782a45aebda66d2e32713def (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.225804 30813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c4ceedcb00a49e1a32977e4e1acea24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.226452 30812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 975ae0ae782a45aebda66d2e32713def. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.232792 30684 tablet_service.cc:1467] Processing CreateTablet for tablet 74c46ec356414d26987e1cd6a0857602 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.236716 30684 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 74c46ec356414d26987e1cd6a0857602. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.244801 31468 ts_tablet_manager.cc:1398] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.030s user 0.003s sys 0.000s
I20240502 14:14:39.245327 31468 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:39.245497 31466 tablet_bootstrap.cc:654] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.245568 31475 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:39.246361 30683 tablet_service.cc:1467] Processing CreateTablet for tablet 5c4ceedcb00a49e1a32977e4e1acea24 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.246860 30683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c4ceedcb00a49e1a32977e4e1acea24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.247324 31468 tablet_bootstrap.cc:654] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.249864 30682 tablet_service.cc:1467] Processing CreateTablet for tablet 975ae0ae782a45aebda66d2e32713def (DEFAULT_TABLE table=rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:39.250613 30682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 975ae0ae782a45aebda66d2e32713def. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:39.261675 31466 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:39.262045 31466 ts_tablet_manager.cc:1367] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.044s user 0.004s sys 0.003s
I20240502 14:14:39.264439 31468 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:39.264468 31475 tablet_bootstrap.cc:654] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.264731 31468 ts_tablet_manager.cc:1367] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.020s user 0.003s sys 0.003s
I20240502 14:14:39.263186 31466 raft_consensus.cc:348] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.265697 31466 raft_consensus.cc:374] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.265802 31466 raft_consensus.cc:729] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.265717 31468 raft_consensus.cc:348] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.266196 31468 raft_consensus.cc:374] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.266098 31466 consensus_queue.cc:260] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.266367 31468 raft_consensus.cc:729] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.266741 31466 ts_tablet_manager.cc:1398] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.005s user 0.001s sys 0.000s
I20240502 14:14:39.266728 31468 consensus_queue.cc:260] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.267072 31466 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:39.267532 31468 ts_tablet_manager.cc:1398] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:39.268994 31466 tablet_bootstrap.cc:654] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.270888 31478 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:39.271047 31475 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:39.271207 31475 ts_tablet_manager.cc:1367] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.026s user 0.000s sys 0.003s
I20240502 14:14:39.272245 31475 raft_consensus.cc:348] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.272492 31475 raft_consensus.cc:374] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.272593 31475 raft_consensus.cc:729] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.272853 31475 consensus_queue.cc:260] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.276151 31478 tablet_bootstrap.cc:654] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.280819 31478 log.cc:824] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:39.280961 31467 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:39.282563 31467 tablet_bootstrap.cc:654] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.283854 31469 ts_tablet_manager.cc:1398] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.070s user 0.012s sys 0.016s
I20240502 14:14:39.284198 31466 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:39.284446 31466 ts_tablet_manager.cc:1367] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.002s
I20240502 14:14:39.284641 31469 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:39.285295 31466 raft_consensus.cc:348] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.285710 31466 raft_consensus.cc:374] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.285863 31466 raft_consensus.cc:729] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.286168 31466 consensus_queue.cc:260] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.286476 31469 tablet_bootstrap.cc:654] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.286902 31466 ts_tablet_manager.cc:1398] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:39.288581 31475 ts_tablet_manager.cc:1398] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.017s user 0.003s sys 0.004s
I20240502 14:14:39.288928 31475 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:39.290573 31475 tablet_bootstrap.cc:654] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.295841 31475 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:39.296041 31475 ts_tablet_manager.cc:1367] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.006s
I20240502 14:14:39.296738 31475 raft_consensus.cc:348] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.297035 31475 raft_consensus.cc:374] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.297132 31475 raft_consensus.cc:729] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.297423 31475 consensus_queue.cc:260] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.297925 31467 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:39.297952 31475 ts_tablet_manager.cc:1398] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:14:39.298151 31467 ts_tablet_manager.cc:1367] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.000s
I20240502 14:14:39.298228 31475 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:39.299347 31467 raft_consensus.cc:348] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.299655 31475 tablet_bootstrap.cc:654] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.299779 31467 raft_consensus.cc:374] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.299942 31467 raft_consensus.cc:729] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.300328 31467 consensus_queue.cc:260] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.301021 31469 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:39.301220 31469 ts_tablet_manager.cc:1367] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.017s user 0.003s sys 0.007s
I20240502 14:14:39.301936 31469 raft_consensus.cc:348] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.302102 31467 ts_tablet_manager.cc:1398] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.004s user 0.002s sys 0.001s
I20240502 14:14:39.302266 31469 raft_consensus.cc:374] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.302373 31469 raft_consensus.cc:729] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.302385 31467 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:39.302671 31469 consensus_queue.cc:260] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.303231 31469 ts_tablet_manager.cc:1398] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:39.305768 31475 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:39.305898 31478 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:39.305963 31475 ts_tablet_manager.cc:1367] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.001s
I20240502 14:14:39.306399 31478 ts_tablet_manager.cc:1367] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.036s user 0.010s sys 0.000s
I20240502 14:14:39.306839 31475 raft_consensus.cc:348] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.307122 31475 raft_consensus.cc:374] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.307147 31467 tablet_bootstrap.cc:654] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.307264 31475 raft_consensus.cc:729] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.307554 31475 consensus_queue.cc:260] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.308084 31475 ts_tablet_manager.cc:1398] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:39.313591 31478 raft_consensus.cc:348] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.314033 31478 raft_consensus.cc:374] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.314126 31478 raft_consensus.cc:729] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.314657 31467 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:39.314821 31467 ts_tablet_manager.cc:1367] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.006s
I20240502 14:14:39.314821 31478 consensus_queue.cc:260] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.315481 31467 raft_consensus.cc:348] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.315722 31467 raft_consensus.cc:374] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.315809 31467 raft_consensus.cc:729] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.316082 31467 consensus_queue.cc:260] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.316622 31478 ts_tablet_manager.cc:1398] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.010s user 0.010s sys 0.000s
I20240502 14:14:39.316815 31467 ts_tablet_manager.cc:1398] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:14:39.317054 31478 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:39.318313 31478 tablet_bootstrap.cc:654] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.320034 31478 tablet_bootstrap.cc:492] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:39.320163 31478 ts_tablet_manager.cc:1367] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:39.320844 31478 raft_consensus.cc:348] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.321009 31478 raft_consensus.cc:374] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.321142 31478 raft_consensus.cc:729] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.321240 31480 raft_consensus.cc:482] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:39.321357 31478 consensus_queue.cc:260] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.321413 31480 raft_consensus.cc:504] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.321913 31478 ts_tablet_manager.cc:1398] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:39.321954 31474 raft_consensus.cc:482] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:39.322156 31478 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:39.322113 31474 raft_consensus.cc:504] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.323457 31480 leader_election.cc:288] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:14:39.323601 31478 tablet_bootstrap.cc:654] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:39.324368 31474 leader_election.cc:288] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:39.329133 31473 raft_consensus.cc:482] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:39.329346 31473 raft_consensus.cc:504] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.333724 31487 raft_consensus.cc:482] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:39.333901 31487 raft_consensus.cc:504] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.337687 31473 leader_election.cc:288] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:39.342219 31478 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:39.344404 31478 ts_tablet_manager.cc:1367] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.022s user 0.002s sys 0.005s
I20240502 14:14:39.345587 31478 raft_consensus.cc:348] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.347512 31487 leader_election.cc:288] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:39.350634 31478 raft_consensus.cc:374] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:39.352551 31478 raft_consensus.cc:729] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:39.353222 31478 consensus_queue.cc:260] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.355108 31478 ts_tablet_manager.cc:1398] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.010s user 0.000s sys 0.002s
I20240502 14:14:39.390964 31489 raft_consensus.cc:482] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:39.391253 31489 raft_consensus.cc:504] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.393416 31224 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:39.393718 31224 raft_consensus.cc:2454] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:39.403286 31489 leader_election.cc:288] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:39.405998 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:39.406477 30574 raft_consensus.cc:2454] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:39.422978 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:39.423249 30834 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:39.433905 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:39.434226 30834 raft_consensus.cc:2454] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:39.434921 31157 leader_election.cc:302] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:39.435487 31474 raft_consensus.cc:2781] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:39.435609 31474 raft_consensus.cc:482] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:39.435693 31474 raft_consensus.cc:3037] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.437348 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:39.437601 30574 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:39.438045 31487 raft_consensus.cc:482] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:39.438248 31487 raft_consensus.cc:504] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.438896 31474 raft_consensus.cc:504] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.439344 31487 leader_election.cc:288] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:39.440568 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:39.440627 31474 leader_election.cc:288] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Requested vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:39.440768 30574 raft_consensus.cc:3037] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.440868 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "74c46ec356414d26987e1cd6a0857602" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:39.441114 30834 raft_consensus.cc:2454] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:39.441247 30899 leader_election.cc:302] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:39.442174 31224 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:39.442394 31224 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:39.442535 31487 raft_consensus.cc:2781] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:39.442626 31487 raft_consensus.cc:482] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:39.442737 31487 raft_consensus.cc:3037] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.442720 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:39.442636 31093 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:39.442973 31094 raft_consensus.cc:2454] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:39.442912 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:39.443356 30704 raft_consensus.cc:2454] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:39.443888 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:39.443972 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:39.444082 30834 raft_consensus.cc:3037] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.444139 30704 raft_consensus.cc:3037] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.445607 31487 raft_consensus.cc:504] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.445796 30574 raft_consensus.cc:2454] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:39.446732 30834 raft_consensus.cc:2454] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:39.447052 31487 leader_election.cc:288] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 election: Requested vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:39.447194 30704 raft_consensus.cc:2454] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:39.447342 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:39.447512 30834 raft_consensus.cc:3037] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.448072 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "74c46ec356414d26987e1cd6a0857602" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:39.448390 30574 raft_consensus.cc:2454] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:39.449152 30899 leader_election.cc:302] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:39.449429 31487 raft_consensus.cc:2781] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:39.449406 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:39.449640 31487 raft_consensus.cc:482] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:39.449669 30574 raft_consensus.cc:3037] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.449798 31487 raft_consensus.cc:3037] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.450203 30834 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:39.450822 30573 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:39.453979 30574 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:39.454335 31487 raft_consensus.cc:504] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.455188 31224 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:39.455317 31157 leader_election.cc:302] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 5 voters: 3 yes votes; 1 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, fd3399b1bfe644159a833ede6726c25f; no voters: ba420f2442d6416bbda95edfff85e849
I20240502 14:14:39.455436 31224 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:39.455636 31474 raft_consensus.cc:2781] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:39.455799 31487 leader_election.cc:288] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 election: Requested vote from peers ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:39.455947 31224 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
I20240502 14:14:39.456041 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "74c46ec356414d26987e1cd6a0857602" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:39.456156 31224 raft_consensus.cc:3037] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.456265 30834 raft_consensus.cc:3037] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.458722 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:39.458910 30834 raft_consensus.cc:2454] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:39.458974 30574 raft_consensus.cc:2379] T 975ae0ae782a45aebda66d2e32713def P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ba420f2442d6416bbda95edfff85e849 in current term 1: Already voted for candidate 0dfd48d0b03a42f689e58d5190d03a2f in this term.
I20240502 14:14:39.459059 31224 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:39.459771 30900 leader_election.cc:302] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3; no voters:
I20240502 14:14:39.460187 31487 raft_consensus.cc:2781] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:39.461670 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:39.461987 30704 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:39.462524 30767 leader_election.cc:302] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 5 voters: 3 yes votes; 1 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, d733c51902044956a8cd2b6b31bc9831; no voters: fd3399b1bfe644159a833ede6726c25f
I20240502 14:14:39.463538 31480 raft_consensus.cc:2741] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader pre-election decision vote started in defunct term 0: won
I20240502 14:14:39.464018 31474 raft_consensus.cc:686] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Becoming Leader. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
I20240502 14:14:39.464613 31474 consensus_queue.cc:237] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.465080 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "74c46ec356414d26987e1cd6a0857602" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:39.465296 30574 raft_consensus.cc:3037] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.470714 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:39.471196 30833 raft_consensus.cc:2379] T 975ae0ae782a45aebda66d2e32713def P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ba420f2442d6416bbda95edfff85e849 in current term 1: Already voted for candidate 0dfd48d0b03a42f689e58d5190d03a2f in this term.
I20240502 14:14:39.471448 30834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "d733c51902044956a8cd2b6b31bc9831" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:39.471701 30834 raft_consensus.cc:2379] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d733c51902044956a8cd2b6b31bc9831 in current term 1: Already voted for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in this term.
I20240502 14:14:39.473451 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:39.475190 31224 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "d733c51902044956a8cd2b6b31bc9831" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:39.475431 31224 raft_consensus.cc:2379] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d733c51902044956a8cd2b6b31bc9831 in current term 1: Already voted for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in this term.
I20240502 14:14:39.476063 30574 raft_consensus.cc:2454] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:39.476027 30963 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "d733c51902044956a8cd2b6b31bc9831" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:39.476737 30899 leader_election.cc:302] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:39.477771 31487 raft_consensus.cc:686] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Becoming Leader. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Running, Role: LEADER
I20240502 14:14:39.478394 31487 consensus_queue.cc:237] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.478528 30637 leader_election.cc:302] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: d733c51902044956a8cd2b6b31bc9831; no voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3
I20240502 14:14:39.478670 31526 raft_consensus.cc:2781] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:39.478952 31489 raft_consensus.cc:3037] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.481791 31093 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "74c46ec356414d26987e1cd6a0857602" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:39.481824 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "74c46ec356414d26987e1cd6a0857602" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:39.482038 31489 raft_consensus.cc:2726] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:14:39.482066 31093 raft_consensus.cc:2454] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:39.482704 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "975ae0ae782a45aebda66d2e32713def" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:39.482511 30431 catalog_manager.cc:5484] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: term changed from 0 to 1, leader changed from <none> to 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134). New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:39.483075 30703 raft_consensus.cc:2379] T 975ae0ae782a45aebda66d2e32713def P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ba420f2442d6416bbda95edfff85e849 in current term 1: Already voted for candidate 0dfd48d0b03a42f689e58d5190d03a2f in this term.
I20240502 14:14:39.484035 31526 raft_consensus.cc:686] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Becoming Leader. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Running, Role: LEADER
I20240502 14:14:39.484236 31027 leader_election.cc:302] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 5 voters: 2 yes votes; 3 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, ba420f2442d6416bbda95edfff85e849; no voters: 155f26c73dc84441a58417c09f577494, d733c51902044956a8cd2b6b31bc9831, fd3399b1bfe644159a833ede6726c25f
I20240502 14:14:39.484390 31526 consensus_queue.cc:237] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:39.484627 31473 raft_consensus.cc:3037] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.485124 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:39.485339 30704 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:39.485800 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "74c46ec356414d26987e1cd6a0857602" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:39.486003 30704 raft_consensus.cc:2454] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:39.486402 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:39.488129 31473 raft_consensus.cc:2726] T 975ae0ae782a45aebda66d2e32713def P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:14:39.489629 30704 raft_consensus.cc:2454] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:39.490841 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "74c46ec356414d26987e1cd6a0857602" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:39.491029 30704 raft_consensus.cc:3037] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.491384 30431 catalog_manager.cc:5484] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:39.493088 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5c4ceedcb00a49e1a32977e4e1acea24" candidate_uuid: "d733c51902044956a8cd2b6b31bc9831" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:39.493343 30574 raft_consensus.cc:2379] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d733c51902044956a8cd2b6b31bc9831 in current term 1: Already voted for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in this term.
I20240502 14:14:39.494417 30704 raft_consensus.cc:2454] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:39.497220 30431 catalog_manager.cc:5484] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } }
W20240502 14:14:39.533257 30750 tablet.cc:2369] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 14:14:39.822961 31531 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:39.823235 31531 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:39.829857 31531 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240502 14:14:39.869486 31474 consensus_queue.cc:1035] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.870327 31093 raft_consensus.cc:3037] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:39.873605 31487 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.875051 31487 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.881184 31487 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.891849 31487 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.893721 31487 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.929355 31558 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 14:14:39.944321 31487 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.944768 31487 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:39.958992 31474 consensus_queue.cc:1035] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:40.005234 31573 consensus_queue.cc:1035] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:40.006186 31573 consensus_queue.cc:1035] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
INSERT report
rows total: 2000
time total: 400.113 ms
time per row: 0.200056 ms
I20240502 14:14:40.394563 30426 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36920:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:14:40.403163 31203 tablet_service.cc:1467] Processing CreateTablet for tablet 23cf772ace1c44ecbd450e0bce820c3f (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.403352 30553 tablet_service.cc:1467] Processing CreateTablet for tablet 23cf772ace1c44ecbd450e0bce820c3f (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.403678 31203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 23cf772ace1c44ecbd450e0bce820c3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.403817 30553 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 23cf772ace1c44ecbd450e0bce820c3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.404172 30944 tablet_service.cc:1467] Processing CreateTablet for tablet 23cf772ace1c44ecbd450e0bce820c3f (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.404585 30944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 23cf772ace1c44ecbd450e0bce820c3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.405115 30683 tablet_service.cc:1467] Processing CreateTablet for tablet 0f906c7d775540e6ba76cf2ff3700a67 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.406214 31204 tablet_service.cc:1467] Processing CreateTablet for tablet 0f906c7d775540e6ba76cf2ff3700a67 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.406596 31204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f906c7d775540e6ba76cf2ff3700a67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.409611 30683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f906c7d775540e6ba76cf2ff3700a67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.409948 31073 tablet_service.cc:1467] Processing CreateTablet for tablet 23cf772ace1c44ecbd450e0bce820c3f (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.410431 31073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 23cf772ace1c44ecbd450e0bce820c3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.414042 30684 tablet_service.cc:1467] Processing CreateTablet for tablet 23cf772ace1c44ecbd450e0bce820c3f (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.414592 30684 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 23cf772ace1c44ecbd450e0bce820c3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.415519 31616 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:40.415812 30813 tablet_service.cc:1467] Processing CreateTablet for tablet 0f906c7d775540e6ba76cf2ff3700a67 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.416388 30813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f906c7d775540e6ba76cf2ff3700a67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.417200 31616 tablet_bootstrap.cc:654] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.418927 30943 tablet_service.cc:1467] Processing CreateTablet for tablet 0f906c7d775540e6ba76cf2ff3700a67 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.419466 30943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f906c7d775540e6ba76cf2ff3700a67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.420996 31074 tablet_service.cc:1467] Processing CreateTablet for tablet 0f906c7d775540e6ba76cf2ff3700a67 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.421386 31074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f906c7d775540e6ba76cf2ff3700a67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.422147 31203 tablet_service.cc:1467] Processing CreateTablet for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.422648 31203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.426940 30942 tablet_service.cc:1467] Processing CreateTablet for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.427460 30942 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.437085 31618 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:40.438367 30682 tablet_service.cc:1467] Processing CreateTablet for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.438856 30682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.440289 31618 tablet_bootstrap.cc:654] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.442498 30554 tablet_service.cc:1467] Processing CreateTablet for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.442997 30554 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.446476 31072 tablet_service.cc:1467] Processing CreateTablet for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 (DEFAULT_TABLE table=rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:40.446998 31072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:40.455297 31620 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:40.458086 31620 tablet_bootstrap.cc:654] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.459625 31616 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:40.459913 31616 ts_tablet_manager.cc:1367] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.044s user 0.000s sys 0.012s
I20240502 14:14:40.460870 31616 raft_consensus.cc:348] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.461370 31616 raft_consensus.cc:374] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.461540 31616 raft_consensus.cc:729] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.461905 31616 consensus_queue.cc:260] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.465128 31616 ts_tablet_manager.cc:1398] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 14:14:40.467602 31616 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:40.469292 31620 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:40.469306 31616 tablet_bootstrap.cc:654] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.469476 31620 ts_tablet_manager.cc:1367] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.005s
I20240502 14:14:40.471033 31620 raft_consensus.cc:348] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.471360 31620 raft_consensus.cc:374] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.472903 31620 raft_consensus.cc:729] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.473500 31620 consensus_queue.cc:260] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.474700 31620 ts_tablet_manager.cc:1398] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
I20240502 14:14:40.476634 31616 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:40.477003 31616 ts_tablet_manager.cc:1367] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.001s
I20240502 14:14:40.482429 31616 raft_consensus.cc:348] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.483449 31620 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:40.483748 31616 raft_consensus.cc:374] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.484004 31616 raft_consensus.cc:729] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.484504 31616 consensus_queue.cc:260] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.485236 31620 tablet_bootstrap.cc:654] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.486634 31621 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:40.488569 31489 raft_consensus.cc:482] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:40.488619 31616 ts_tablet_manager.cc:1398] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.007s user 0.001s sys 0.001s
I20240502 14:14:40.488760 31489 raft_consensus.cc:504] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.488966 31621 tablet_bootstrap.cc:654] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.490051 31489 leader_election.cc:288] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:40.490998 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "d733c51902044956a8cd2b6b31bc9831" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:40.491030 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "d733c51902044956a8cd2b6b31bc9831" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:40.491246 31223 raft_consensus.cc:2454] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d733c51902044956a8cd2b6b31bc9831 in term 0.
W20240502 14:14:40.492050 30637 leader_election.cc:341] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:40.492403 31616 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:40.492504 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "d733c51902044956a8cd2b6b31bc9831" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:40.494338 31616 tablet_bootstrap.cc:654] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
W20240502 14:14:40.496064 30638 leader_election.cc:341] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:40.498207 31618 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:40.498432 31618 ts_tablet_manager.cc:1367] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.061s user 0.005s sys 0.007s
I20240502 14:14:40.499373 31618 raft_consensus.cc:348] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.499780 31618 raft_consensus.cc:374] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.499934 31618 raft_consensus.cc:729] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.500301 31618 consensus_queue.cc:260] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.504052 31622 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:40.504333 31620 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:40.504560 31620 ts_tablet_manager.cc:1367] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.021s user 0.000s sys 0.004s
I20240502 14:14:40.506332 31618 ts_tablet_manager.cc:1398] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 14:14:40.506865 31618 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:40.505936 31620 raft_consensus.cc:348] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.509091 31092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "d733c51902044956a8cd2b6b31bc9831" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
W20240502 14:14:40.510351 30638 leader_election.cc:341] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:40.510524 31620 raft_consensus.cc:374] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.510597 30638 leader_election.cc:302] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 5 voters: 2 yes votes; 3 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, d733c51902044956a8cd2b6b31bc9831; no voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, ba420f2442d6416bbda95edfff85e849
I20240502 14:14:40.511130 31489 raft_consensus.cc:2726] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:14:40.511155 31615 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:40.511443 31622 tablet_bootstrap.cc:654] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.510618 31620 raft_consensus.cc:729] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.511868 31620 consensus_queue.cc:260] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.512244 31618 tablet_bootstrap.cc:654] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.512758 31620 ts_tablet_manager.cc:1398] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.008s user 0.000s sys 0.001s
I20240502 14:14:40.512969 31615 tablet_bootstrap.cc:654] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.513123 31620 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:40.514127 31621 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:40.516299 31620 tablet_bootstrap.cc:654] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.516582 31621 ts_tablet_manager.cc:1367] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.030s user 0.007s sys 0.016s
I20240502 14:14:40.517675 31621 raft_consensus.cc:348] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.518815 31621 raft_consensus.cc:374] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.518975 31621 raft_consensus.cc:729] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.519373 31621 consensus_queue.cc:260] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.521824 31615 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:40.522022 31616 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:40.522085 31615 ts_tablet_manager.cc:1367] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.011s user 0.005s sys 0.005s
I20240502 14:14:40.522238 31616 ts_tablet_manager.cc:1367] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.030s user 0.005s sys 0.003s
I20240502 14:14:40.522549 31618 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:40.522727 31618 ts_tablet_manager.cc:1367] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.000s
I20240502 14:14:40.523202 31616 raft_consensus.cc:348] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.523175 31615 raft_consensus.cc:348] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.523536 31616 raft_consensus.cc:374] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.523612 31615 raft_consensus.cc:374] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.523489 31618 raft_consensus.cc:348] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.523679 31616 raft_consensus.cc:729] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.523741 31618 raft_consensus.cc:374] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.523824 31618 raft_consensus.cc:729] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.523834 31615 raft_consensus.cc:729] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.524134 31616 consensus_queue.cc:260] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.524264 31615 consensus_queue.cc:260] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.528148 31618 consensus_queue.cc:260] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.528725 31618 ts_tablet_manager.cc:1398] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 14:14:40.528993 31618 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:40.529878 31615 ts_tablet_manager.cc:1398] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
I20240502 14:14:40.531363 31615 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:40.531507 31621 ts_tablet_manager.cc:1398] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.015s user 0.005s sys 0.000s
I20240502 14:14:40.533835 31618 tablet_bootstrap.cc:654] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.534808 31615 tablet_bootstrap.cc:654] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.536162 31620 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:40.536340 31620 ts_tablet_manager.cc:1367] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.023s user 0.001s sys 0.006s
I20240502 14:14:40.537250 31620 raft_consensus.cc:348] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.537571 31620 raft_consensus.cc:374] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.537734 31620 raft_consensus.cc:729] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.538100 31620 consensus_queue.cc:260] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.538461 31622 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:40.538755 31622 ts_tablet_manager.cc:1367] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.035s user 0.012s sys 0.009s
I20240502 14:14:40.538847 31620 ts_tablet_manager.cc:1398] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 14:14:40.539959 31622 raft_consensus.cc:348] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.540581 31622 raft_consensus.cc:374] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.540807 31622 raft_consensus.cc:729] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.541220 31622 consensus_queue.cc:260] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.545115 31622 ts_tablet_manager.cc:1398] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.006s user 0.005s sys 0.000s
I20240502 14:14:40.545346 31615 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:40.545594 31615 ts_tablet_manager.cc:1367] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.014s user 0.010s sys 0.000s
I20240502 14:14:40.545887 31622 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:40.546468 31615 raft_consensus.cc:348] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.546828 31615 raft_consensus.cc:374] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.546952 31615 raft_consensus.cc:729] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.547281 31615 consensus_queue.cc:260] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.547602 31622 tablet_bootstrap.cc:654] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.548053 31615 ts_tablet_manager.cc:1398] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:40.551313 31571 raft_consensus.cc:482] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:40.551538 31571 raft_consensus.cc:504] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.553210 31571 leader_election.cc:288] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259)
I20240502 14:14:40.553498 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:40.553809 30572 raft_consensus.cc:2454] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:40.554744 31616 ts_tablet_manager.cc:1398] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.032s user 0.002s sys 0.000s
I20240502 14:14:40.556510 31618 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:40.556567 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:40.556738 31618 ts_tablet_manager.cc:1367] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.028s user 0.013s sys 0.006s
I20240502 14:14:40.556846 30703 raft_consensus.cc:2454] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:40.557677 31157 leader_election.cc:302] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, d733c51902044956a8cd2b6b31bc9831, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:40.557726 31618 raft_consensus.cc:348] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.557971 31618 raft_consensus.cc:374] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.558107 31618 raft_consensus.cc:729] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.558422 31618 consensus_queue.cc:260] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.558725 31571 raft_consensus.cc:2781] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:40.558890 31571 raft_consensus.cc:482] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:40.559013 31571 raft_consensus.cc:3037] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.559223 31618 ts_tablet_manager.cc:1398] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:40.563498 31571 raft_consensus.cc:504] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.565137 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:40.565405 30572 raft_consensus.cc:3037] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.565394 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:40.565667 30703 raft_consensus.cc:3037] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.569389 30572 raft_consensus.cc:2454] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:40.569411 30703 raft_consensus.cc:2454] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:40.564884 31571 leader_election.cc:288] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Requested vote from peers fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259)
I20240502 14:14:40.570132 31157 leader_election.cc:302] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, d733c51902044956a8cd2b6b31bc9831, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:40.570457 31571 raft_consensus.cc:2781] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:40.570638 31571 raft_consensus.cc:686] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Becoming Leader. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
I20240502 14:14:40.570966 31571 consensus_queue.cc:237] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.573428 31092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:40.574815 31092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
W20240502 14:14:40.576043 31158 leader_election.cc:341] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805): Illegal state: must be running to vote when last-logged opid is not known
W20240502 14:14:40.576504 31158 leader_election.cc:341] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:40.578013 31622 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:40.578244 31622 ts_tablet_manager.cc:1367] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.033s user 0.009s sys 0.021s
I20240502 14:14:40.579427 31622 raft_consensus.cc:348] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.579766 31622 raft_consensus.cc:374] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.579869 31622 raft_consensus.cc:729] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.580150 31622 consensus_queue.cc:260] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.581806 31622 ts_tablet_manager.cc:1398] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:40.582135 31622 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:40.583765 31622 tablet_bootstrap.cc:654] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:40.593071 31610 raft_consensus.cc:482] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:40.593322 31610 raft_consensus.cc:504] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.595016 31610 leader_election.cc:288] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:40.596751 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4aa1eb08d4b450f9c5be499d8c6bbc7" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:40.596750 31092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4aa1eb08d4b450f9c5be499d8c6bbc7" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:40.596995 31092 raft_consensus.cc:2454] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:40.596995 30572 raft_consensus.cc:2454] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:40.597388 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4aa1eb08d4b450f9c5be499d8c6bbc7" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:40.597735 30703 raft_consensus.cc:2454] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:40.598662 31159 leader_election.cc:302] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:40.599411 31610 raft_consensus.cc:2781] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:40.599556 31610 raft_consensus.cc:482] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:40.599669 31610 raft_consensus.cc:3037] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.603662 31610 raft_consensus.cc:504] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.605365 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:40.605388 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4aa1eb08d4b450f9c5be499d8c6bbc7" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:40.605685 30703 raft_consensus.cc:3037] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.605660 30963 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:40.606181 30962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4aa1eb08d4b450f9c5be499d8c6bbc7" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:40.606515 30962 raft_consensus.cc:2454] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:40.605654 30964 raft_consensus.cc:2454] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:40.607410 30961 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4aa1eb08d4b450f9c5be499d8c6bbc7" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:40.607698 30961 raft_consensus.cc:3037] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.609395 30703 raft_consensus.cc:2454] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:40.610884 31610 leader_election.cc:288] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Requested vote from peers 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:40.610854 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4aa1eb08d4b450f9c5be499d8c6bbc7" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:40.611088 30572 raft_consensus.cc:3037] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.611429 30961 raft_consensus.cc:2454] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:40.613584 30426 catalog_manager.cc:5484] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: term changed from 0 to 1, leader changed from <none> to 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134). New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:40.613880 30572 raft_consensus.cc:2454] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:40.614254 31159 leader_election.cc:302] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, d733c51902044956a8cd2b6b31bc9831, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:40.614531 31571 raft_consensus.cc:2781] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:40.614725 31571 raft_consensus.cc:686] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Becoming Leader. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
I20240502 14:14:40.615011 31571 consensus_queue.cc:237] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:40.616266 31092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4aa1eb08d4b450f9c5be499d8c6bbc7" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:40.616633 31092 raft_consensus.cc:3037] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.619648 31622 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:40.619752 31092 raft_consensus.cc:2454] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:40.621026 31622 ts_tablet_manager.cc:1367] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.039s user 0.011s sys 0.017s
I20240502 14:14:40.621727 30426 catalog_manager.cc:5484] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: term changed from 0 to 1, leader changed from <none> to 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134). New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:40.622720 31622 raft_consensus.cc:348] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.622931 31622 raft_consensus.cc:374] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:40.623024 31622 raft_consensus.cc:729] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:40.625691 31622 consensus_queue.cc:260] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.626714 31622 ts_tablet_manager.cc:1398] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
I20240502 14:14:40.684031 31634 raft_consensus.cc:482] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:40.684278 31634 raft_consensus.cc:504] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.685164 31634 leader_election.cc:288] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:40.685730 30961 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:40.685740 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:40.685843 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:40.685977 30961 raft_consensus.cc:2454] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:40.686017 30703 raft_consensus.cc:2454] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:40.686056 31223 raft_consensus.cc:2454] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:40.686666 30767 leader_election.cc:302] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, d733c51902044956a8cd2b6b31bc9831; no voters:
I20240502 14:14:40.686967 31634 raft_consensus.cc:2781] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:40.687065 31634 raft_consensus.cc:482] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:40.687179 31634 raft_consensus.cc:3037] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.690284 31634 raft_consensus.cc:504] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.691665 31634 leader_election.cc:288] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 election: Requested vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:40.691646 30961 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:40.691702 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:40.691897 30961 raft_consensus.cc:3037] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.691928 30703 raft_consensus.cc:3037] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.692584 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
I20240502 14:14:40.692793 31223 raft_consensus.cc:3037] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.694805 30703 raft_consensus.cc:2454] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 1.
I20240502 14:14:40.694898 30961 raft_consensus.cc:2454] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 1.
I20240502 14:14:40.695488 31092 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:40.695570 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:40.695691 31223 raft_consensus.cc:2454] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 1.
I20240502 14:14:40.695780 31092 raft_consensus.cc:2454] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:40.696105 30770 leader_election.cc:302] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3; no voters:
I20240502 14:14:40.696784 31634 raft_consensus.cc:2781] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:40.697295 31634 raft_consensus.cc:686] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 LEADER]: Becoming Leader. State: Replica: 155f26c73dc84441a58417c09f577494, State: Running, Role: LEADER
I20240502 14:14:40.697961 31634 consensus_queue.cc:237] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } }
I20240502 14:14:40.701726 30426 catalog_manager.cc:5484] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 reported cstate change: term changed from 0 to 1, leader changed from <none> to 155f26c73dc84441a58417c09f577494 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "155f26c73dc84441a58417c09f577494" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:40.892323 31637 raft_consensus.cc:482] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:40.892514 31637 raft_consensus.cc:504] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:40.893553 31637 leader_election.cc:288] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259)
I20240502 14:14:40.894030 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:40.894340 30572 raft_consensus.cc:2379] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ba420f2442d6416bbda95edfff85e849 in current term 1: Already voted for candidate 0dfd48d0b03a42f689e58d5190d03a2f in this term.
I20240502 14:14:40.894297 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:40.894479 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:40.894801 30703 raft_consensus.cc:2379] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ba420f2442d6416bbda95edfff85e849 in current term 1: Already voted for candidate 0dfd48d0b03a42f689e58d5190d03a2f in this term.
I20240502 14:14:40.895391 31030 leader_election.cc:302] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: ba420f2442d6416bbda95edfff85e849; no voters: 0dfd48d0b03a42f689e58d5190d03a2f, d733c51902044956a8cd2b6b31bc9831, fd3399b1bfe644159a833ede6726c25f
I20240502 14:14:40.895712 31637 raft_consensus.cc:3037] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.898099 31637 raft_consensus.cc:2726] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:14:40.901404 30961 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "23cf772ace1c44ecbd450e0bce820c3f" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:40.901630 30961 raft_consensus.cc:2454] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:40.965126 31571 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:40.994187 30961 raft_consensus.cc:3037] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:40.994865 31610 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:40.997786 31610 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:41.001734 31571 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:41.008119 31571 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:41.065656 31610 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:41.071532 31610 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:41.075047 31610 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:14:41.107975 31646 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:41.108369 31646 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:41.120522 31646 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 14:14:41.169162 31634 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:41.202450 31634 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:41.216241 31092 raft_consensus.cc:3037] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.221529 31634 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:41.222258 31634 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:14:41.351090 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 43516.9us
INSERT report
rows total: 2000
time total: 376.701 ms
time per row: 0.188351 ms
I20240502 14:14:41.641022 30430 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36920:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:14:41.649967 30682 tablet_service.cc:1467] Processing CreateTablet for tablet 7e5aa36e1a914374807e91f44bb1cff9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.650195 30943 tablet_service.cc:1467] Processing CreateTablet for tablet 7e5aa36e1a914374807e91f44bb1cff9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.650449 30682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7e5aa36e1a914374807e91f44bb1cff9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.650640 30943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7e5aa36e1a914374807e91f44bb1cff9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.650605 30813 tablet_service.cc:1467] Processing CreateTablet for tablet 7e5aa36e1a914374807e91f44bb1cff9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.651049 30813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7e5aa36e1a914374807e91f44bb1cff9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.653762 30553 tablet_service.cc:1467] Processing CreateTablet for tablet 7e5aa36e1a914374807e91f44bb1cff9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.654207 30553 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7e5aa36e1a914374807e91f44bb1cff9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.654371 30552 tablet_service.cc:1467] Processing CreateTablet for tablet 8b1b5e519c314a83bf1e39e011440fa6 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.654796 30552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b1b5e519c314a83bf1e39e011440fa6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.655273 30814 tablet_service.cc:1467] Processing CreateTablet for tablet 356c1606c56147a2868e60ab63c59c13 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.655458 30684 tablet_service.cc:1467] Processing CreateTablet for tablet 356c1606c56147a2868e60ab63c59c13 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.655593 30814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 356c1606c56147a2868e60ab63c59c13. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.655810 30684 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 356c1606c56147a2868e60ab63c59c13. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.654345 30554 tablet_service.cc:1467] Processing CreateTablet for tablet 356c1606c56147a2868e60ab63c59c13 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.658253 30554 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 356c1606c56147a2868e60ab63c59c13. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.660429 30812 tablet_service.cc:1467] Processing CreateTablet for tablet 8b1b5e519c314a83bf1e39e011440fa6 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.660761 30812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b1b5e519c314a83bf1e39e011440fa6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.662397 30942 tablet_service.cc:1467] Processing CreateTablet for tablet 356c1606c56147a2868e60ab63c59c13 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.662808 30942 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 356c1606c56147a2868e60ab63c59c13. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.662758 31072 tablet_service.cc:1467] Processing CreateTablet for tablet 7e5aa36e1a914374807e91f44bb1cff9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.662850 31073 tablet_service.cc:1467] Processing CreateTablet for tablet 8b1b5e519c314a83bf1e39e011440fa6 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.663201 31072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7e5aa36e1a914374807e91f44bb1cff9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.665911 30683 tablet_service.cc:1467] Processing CreateTablet for tablet 8b1b5e519c314a83bf1e39e011440fa6 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.666272 30683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b1b5e519c314a83bf1e39e011440fa6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.666337 31073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b1b5e519c314a83bf1e39e011440fa6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.670051 31203 tablet_service.cc:1467] Processing CreateTablet for tablet 356c1606c56147a2868e60ab63c59c13 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.670514 31203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 356c1606c56147a2868e60ab63c59c13. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.673941 31204 tablet_service.cc:1467] Processing CreateTablet for tablet 8b1b5e519c314a83bf1e39e011440fa6 (DEFAULT_TABLE table=rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:41.674353 31204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b1b5e519c314a83bf1e39e011440fa6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:41.681429 31738 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:41.681777 31739 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:41.683442 31738 tablet_bootstrap.cc:654] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.683744 31739 tablet_bootstrap.cc:654] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.687582 31742 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:41.689412 31742 tablet_bootstrap.cc:654] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.690393 31737 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:41.692615 31737 tablet_bootstrap.cc:654] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.699299 31745 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:41.701296 31745 tablet_bootstrap.cc:654] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.702522 31738 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:41.702734 31738 ts_tablet_manager.cc:1367] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.021s user 0.006s sys 0.002s
I20240502 14:14:41.703629 31736 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:41.704579 31742 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:41.703572 31738 raft_consensus.cc:348] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.705005 31742 ts_tablet_manager.cc:1367] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.018s user 0.006s sys 0.000s
I20240502 14:14:41.705129 31738 raft_consensus.cc:374] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.705422 31736 tablet_bootstrap.cc:654] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.706051 31742 raft_consensus.cc:348] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.706342 31738 raft_consensus.cc:729] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.706578 31745 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:41.706828 31745 ts_tablet_manager.cc:1367] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.008s user 0.004s sys 0.000s
I20240502 14:14:41.706749 31738 consensus_queue.cc:260] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.707804 31738 ts_tablet_manager.cc:1398] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.005s user 0.002s sys 0.001s
I20240502 14:14:41.707971 31742 raft_consensus.cc:374] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.708139 31742 raft_consensus.cc:729] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.708513 31742 consensus_queue.cc:260] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.708729 31745 raft_consensus.cc:348] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.709056 31745 raft_consensus.cc:374] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.709216 31745 raft_consensus.cc:729] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.709300 31738 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:41.710461 31737 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:41.710654 31737 ts_tablet_manager.cc:1367] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.020s user 0.005s sys 0.000s
I20240502 14:14:41.711064 31738 tablet_bootstrap.cc:654] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.711642 31737 raft_consensus.cc:348] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.712033 31737 raft_consensus.cc:374] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.712188 31737 raft_consensus.cc:729] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.709553 31745 consensus_queue.cc:260] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.712566 31737 consensus_queue.cc:260] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.713557 31742 ts_tablet_manager.cc:1398] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
I20240502 14:14:41.713935 31742 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:41.716001 31742 tablet_bootstrap.cc:654] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.717777 31736 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:41.717978 31736 ts_tablet_manager.cc:1367] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.014s user 0.008s sys 0.001s
I20240502 14:14:41.718909 31736 raft_consensus.cc:348] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.719203 31736 raft_consensus.cc:374] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.719352 31736 raft_consensus.cc:729] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.719344 31745 ts_tablet_manager.cc:1398] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.012s user 0.002s sys 0.000s
I20240502 14:14:41.719667 31736 consensus_queue.cc:260] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.721129 31736 ts_tablet_manager.cc:1398] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:14:41.721479 31736 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:41.722134 31745 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:41.724529 31745 tablet_bootstrap.cc:654] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.724987 31739 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:41.725203 31739 ts_tablet_manager.cc:1367] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.044s user 0.012s sys 0.021s
I20240502 14:14:41.725991 31736 tablet_bootstrap.cc:654] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.726262 31738 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:41.726256 31739 raft_consensus.cc:348] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.726481 31738 ts_tablet_manager.cc:1367] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.000s
I20240502 14:14:41.726644 31739 raft_consensus.cc:374] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.726804 31739 raft_consensus.cc:729] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.727146 31737 ts_tablet_manager.cc:1398] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.016s user 0.002s sys 0.002s
I20240502 14:14:41.727190 31739 consensus_queue.cc:260] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.727556 31737 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:41.727250 31738 raft_consensus.cc:348] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.728096 31738 raft_consensus.cc:374] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.728214 31738 raft_consensus.cc:729] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.728591 31739 ts_tablet_manager.cc:1398] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:14:41.728574 31738 consensus_queue.cc:260] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.728983 31739 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:41.729233 31737 tablet_bootstrap.cc:654] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.729367 31738 ts_tablet_manager.cc:1398] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 14:14:41.732723 31739 tablet_bootstrap.cc:654] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.733799 31736 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:41.734031 31736 ts_tablet_manager.cc:1367] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.002s
I20240502 14:14:41.734830 31736 raft_consensus.cc:348] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.735057 31736 raft_consensus.cc:374] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.735138 31736 raft_consensus.cc:729] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.735671 31736 consensus_queue.cc:260] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.736104 31742 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:41.736336 31742 ts_tablet_manager.cc:1367] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.022s user 0.001s sys 0.006s
I20240502 14:14:41.737221 31742 raft_consensus.cc:348] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.737612 31742 raft_consensus.cc:374] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.737742 31742 raft_consensus.cc:729] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.738111 31742 consensus_queue.cc:260] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.738801 31745 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:41.738802 31736 ts_tablet_manager.cc:1398] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 14:14:41.739048 31745 ts_tablet_manager.cc:1367] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.017s user 0.002s sys 0.007s
I20240502 14:14:41.739985 31736 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:41.740651 31745 raft_consensus.cc:348] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.741016 31737 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:41.741096 31745 raft_consensus.cc:374] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.741248 31737 ts_tablet_manager.cc:1367] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.014s user 0.000s sys 0.005s
I20240502 14:14:41.741262 31745 raft_consensus.cc:729] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.741972 31742 ts_tablet_manager.cc:1398] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
I20240502 14:14:41.741667 31745 consensus_queue.cc:260] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.742535 31737 raft_consensus.cc:348] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.742835 31737 raft_consensus.cc:374] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.742928 31737 raft_consensus.cc:729] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.743084 31745 ts_tablet_manager.cc:1398] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.004s user 0.001s sys 0.002s
I20240502 14:14:41.743202 31737 consensus_queue.cc:260] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.744000 31737 ts_tablet_manager.cc:1398] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:14:41.745301 31742 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:41.745724 31739 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:41.745769 31736 tablet_bootstrap.cc:654] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.745929 31739 ts_tablet_manager.cc:1367] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.017s user 0.009s sys 0.000s
I20240502 14:14:41.746721 31739 raft_consensus.cc:348] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.746909 31737 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:41.747017 31739 raft_consensus.cc:374] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.747097 31739 raft_consensus.cc:729] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.747323 31739 consensus_queue.cc:260] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.747910 31739 ts_tablet_manager.cc:1398] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:14:41.748382 31737 tablet_bootstrap.cc:654] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.749289 31742 tablet_bootstrap.cc:654] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:41.749991 31686 raft_consensus.cc:482] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:41.750195 31686 raft_consensus.cc:504] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.751417 31686 leader_election.cc:288] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:41.752333 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5aa36e1a914374807e91f44bb1cff9" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:41.752650 30964 raft_consensus.cc:2454] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:41.753108 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5aa36e1a914374807e91f44bb1cff9" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:41.753177 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5aa36e1a914374807e91f44bb1cff9" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:41.753389 31094 raft_consensus.cc:2454] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:41.753434 30704 raft_consensus.cc:2454] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 0.
I20240502 14:14:41.754321 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5aa36e1a914374807e91f44bb1cff9" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:41.754732 30768 leader_election.cc:302] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, ba420f2442d6416bbda95edfff85e849; no voters:
W20240502 14:14:41.755363 30769 leader_election.cc:341] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:14:41.756356 31686 raft_consensus.cc:2781] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:41.756477 31686 raft_consensus.cc:482] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:41.756558 31686 raft_consensus.cc:3037] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.757380 31736 tablet_bootstrap.cc:492] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:41.757506 31736 ts_tablet_manager.cc:1367] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.003s
I20240502 14:14:41.758200 31736 raft_consensus.cc:348] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.758440 31736 raft_consensus.cc:374] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.758517 31736 raft_consensus.cc:729] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.758770 31736 consensus_queue.cc:260] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.759783 31686 raft_consensus.cc:504] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.760251 31737 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:41.760397 31737 ts_tablet_manager.cc:1367] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.014s user 0.004s sys 0.002s
I20240502 14:14:41.760592 31742 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:41.760742 31742 ts_tablet_manager.cc:1367] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.006s
I20240502 14:14:41.761193 31737 raft_consensus.cc:348] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.761291 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5aa36e1a914374807e91f44bb1cff9" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:41.761471 31737 raft_consensus.cc:374] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.761596 30704 raft_consensus.cc:3037] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.761600 31737 raft_consensus.cc:729] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.761740 31736 ts_tablet_manager.cc:1398] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:14:41.761513 31742 raft_consensus.cc:348] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.761859 31742 raft_consensus.cc:374] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:41.761945 31742 raft_consensus.cc:729] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:41.761884 31737 consensus_queue.cc:260] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.762172 31742 consensus_queue.cc:260] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.762482 31737 ts_tablet_manager.cc:1398] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:41.762679 31742 ts_tablet_manager.cc:1398] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:14:41.764088 30704 raft_consensus.cc:2454] T 7e5aa36e1a914374807e91f44bb1cff9 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 1.
I20240502 14:14:41.765390 31686 leader_election.cc:288] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 election: Requested vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:41.765555 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5aa36e1a914374807e91f44bb1cff9" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:41.765729 30964 raft_consensus.cc:3037] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.766450 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5aa36e1a914374807e91f44bb1cff9" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:41.766595 30574 raft_consensus.cc:3037] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.767884 30964 raft_consensus.cc:2454] T 7e5aa36e1a914374807e91f44bb1cff9 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 1.
I20240502 14:14:41.768285 30768 leader_election.cc:302] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, d733c51902044956a8cd2b6b31bc9831; no voters:
I20240502 14:14:41.768860 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7e5aa36e1a914374807e91f44bb1cff9" candidate_uuid: "155f26c73dc84441a58417c09f577494" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:41.769084 31094 raft_consensus.cc:3037] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.769127 30574 raft_consensus.cc:2454] T 7e5aa36e1a914374807e91f44bb1cff9 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 1.
I20240502 14:14:41.769634 31686 raft_consensus.cc:2781] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:41.769850 31686 raft_consensus.cc:686] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [term 1 LEADER]: Becoming Leader. State: Replica: 155f26c73dc84441a58417c09f577494, State: Running, Role: LEADER
I20240502 14:14:41.770110 31686 consensus_queue.cc:237] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:41.771461 31094 raft_consensus.cc:2454] T 7e5aa36e1a914374807e91f44bb1cff9 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 155f26c73dc84441a58417c09f577494 in term 1.
I20240502 14:14:41.773502 30430 catalog_manager.cc:5484] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 reported cstate change: term changed from 0 to 1, leader changed from <none> to 155f26c73dc84441a58417c09f577494 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "155f26c73dc84441a58417c09f577494" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:41.830425 31704 raft_consensus.cc:482] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:41.830606 31704 raft_consensus.cc:504] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.831452 31704 leader_election.cc:288] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259)
I20240502 14:14:41.831845 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b1b5e519c314a83bf1e39e011440fa6" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:41.831916 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b1b5e519c314a83bf1e39e011440fa6" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:41.832144 31094 raft_consensus.cc:2454] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:41.832190 30574 raft_consensus.cc:2454] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:41.832408 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b1b5e519c314a83bf1e39e011440fa6" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:41.832676 30833 raft_consensus.cc:2454] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:41.832908 31159 leader_election.cc:302] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:41.833077 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b1b5e519c314a83bf1e39e011440fa6" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:41.833235 31701 raft_consensus.cc:2781] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:41.833328 31701 raft_consensus.cc:482] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:41.833307 30704 raft_consensus.cc:2454] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:41.833436 31701 raft_consensus.cc:3037] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.835356 31701 raft_consensus.cc:504] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.836241 31701 leader_election.cc:288] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Requested vote from peers ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259)
I20240502 14:14:41.836465 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b1b5e519c314a83bf1e39e011440fa6" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:41.836546 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b1b5e519c314a83bf1e39e011440fa6" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:41.836705 30574 raft_consensus.cc:3037] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.836863 31094 raft_consensus.cc:3037] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.837023 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b1b5e519c314a83bf1e39e011440fa6" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:41.837213 30833 raft_consensus.cc:3037] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.839376 30833 raft_consensus.cc:2454] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:41.839460 30574 raft_consensus.cc:2454] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:41.839613 31701 raft_consensus.cc:482] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:41.839830 31094 raft_consensus.cc:2454] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:41.839779 31701 raft_consensus.cc:504] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.837105 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b1b5e519c314a83bf1e39e011440fa6" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:41.840123 31159 leader_election.cc:302] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:41.840237 30704 raft_consensus.cc:3037] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.840508 31704 raft_consensus.cc:2781] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:41.840667 31704 raft_consensus.cc:686] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Becoming Leader. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
I20240502 14:14:41.840819 31701 leader_election.cc:288] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:14:41.841027 31704 consensus_queue.cc:237] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } }
I20240502 14:14:41.841727 30702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:41.841946 30702 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:41.842223 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:41.842463 30964 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:41.842823 31158 leader_election.cc:302] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 7c1f568f8efc4b2a9dbf38907f02f3e3, d733c51902044956a8cd2b6b31bc9831; no voters:
I20240502 14:14:41.842813 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:41.842996 30574 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:41.843125 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:41.843276 30704 raft_consensus.cc:2454] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:41.843319 30833 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:41.843725 31704 raft_consensus.cc:2781] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:41.843829 31704 raft_consensus.cc:482] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:41.843963 31704 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.845495 30430 catalog_manager.cc:5484] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: term changed from 0 to 1, leader changed from <none> to 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134). New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:41.848733 31704 raft_consensus.cc:504] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.849882 30574 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:41.849977 30704 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:41.850135 30574 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.850100 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:41.850294 30964 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.850315 30704 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.852922 30574 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:41.853042 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:41.853253 30833 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:41.853292 31704 leader_election.cc:288] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Requested vote from peers fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:14:41.853142 30964 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:41.853253 30704 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:41.853760 31157 leader_election.cc:302] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, d733c51902044956a8cd2b6b31bc9831, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:41.854053 31704 raft_consensus.cc:2781] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:41.854224 31704 raft_consensus.cc:686] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Becoming Leader. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
I20240502 14:14:41.854480 31704 consensus_queue.cc:237] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:41.855877 30833 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:41.857502 30430 catalog_manager.cc:5484] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: term changed from 0 to 1, leader changed from <none> to 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134). New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:42.193729 31686 consensus_queue.cc:1035] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.203802 31686 consensus_queue.cc:1035] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.210016 31686 consensus_queue.cc:1035] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.224118 31689 consensus_queue.cc:1035] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
W20240502 14:14:42.246040 31761 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:42.246393 31761 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:42.254164 31761 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:14:42.263561 31704 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.267808 31704 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.274307 31702 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.282277 31784 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
I20240502 14:14:42.334331 31784 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.346980 31784 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.365357 31702 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:42.367257 31702 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 387.706 ms
time per row: 0.193853 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 14:14:43.758972 30431 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36920:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:14:43.768033 30813 tablet_service.cc:1467] Processing CreateTablet for tablet 50f7b19e515e497c9a27e8e94cb73079 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.768020 30943 tablet_service.cc:1467] Processing CreateTablet for tablet 50f7b19e515e497c9a27e8e94cb73079 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.768471 30813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f7b19e515e497c9a27e8e94cb73079. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.768502 30943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f7b19e515e497c9a27e8e94cb73079. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.769430 31204 tablet_service.cc:1467] Processing CreateTablet for tablet 50f7b19e515e497c9a27e8e94cb73079 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.769649 30552 tablet_service.cc:1467] Processing CreateTablet for tablet 50f7b19e515e497c9a27e8e94cb73079 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.769901 31204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f7b19e515e497c9a27e8e94cb73079. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.770051 30552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f7b19e515e497c9a27e8e94cb73079. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.774152 31203 tablet_service.cc:1467] Processing CreateTablet for tablet 3183d887deec4e40b88dceda3306c0f1 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.774520 31203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3183d887deec4e40b88dceda3306c0f1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.778103 30682 tablet_service.cc:1467] Processing CreateTablet for tablet 3183d887deec4e40b88dceda3306c0f1 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.778643 30682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3183d887deec4e40b88dceda3306c0f1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.782605 31072 tablet_service.cc:1467] Processing CreateTablet for tablet 50f7b19e515e497c9a27e8e94cb73079 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.782706 30942 tablet_service.cc:1467] Processing CreateTablet for tablet 3183d887deec4e40b88dceda3306c0f1 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.783030 31072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f7b19e515e497c9a27e8e94cb73079. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.783207 30942 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3183d887deec4e40b88dceda3306c0f1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.783555 30812 tablet_service.cc:1467] Processing CreateTablet for tablet 3183d887deec4e40b88dceda3306c0f1 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.783936 30812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3183d887deec4e40b88dceda3306c0f1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.785111 31848 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:43.786706 31848 tablet_bootstrap.cc:654] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.789798 30814 tablet_service.cc:1467] Processing CreateTablet for tablet 3c26d346223a498bb96b962fa7dd99aa (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.790263 30814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c26d346223a498bb96b962fa7dd99aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.794725 31073 tablet_service.cc:1467] Processing CreateTablet for tablet 3183d887deec4e40b88dceda3306c0f1 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.795307 31073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3183d887deec4e40b88dceda3306c0f1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.795950 30552 tablet_service.cc:1467] Processing CreateTablet for tablet 3c26d346223a498bb96b962fa7dd99aa (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.796433 30552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c26d346223a498bb96b962fa7dd99aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.797708 31847 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:43.799328 31847 tablet_bootstrap.cc:654] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.803305 31202 tablet_service.cc:1467] Processing CreateTablet for tablet 3c26d346223a498bb96b962fa7dd99aa (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.803550 31072 tablet_service.cc:1467] Processing CreateTablet for tablet 3c26d346223a498bb96b962fa7dd99aa (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.803901 31202 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c26d346223a498bb96b962fa7dd99aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.803912 31072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c26d346223a498bb96b962fa7dd99aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.807461 30943 tablet_service.cc:1467] Processing CreateTablet for tablet 3c26d346223a498bb96b962fa7dd99aa (DEFAULT_TABLE table=rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:43.807854 30943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c26d346223a498bb96b962fa7dd99aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:43.813198 31850 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:43.815014 31850 tablet_bootstrap.cc:654] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.816468 31852 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:43.822779 31854 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:43.823542 31846 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:43.823993 31850 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:43.824208 31850 ts_tablet_manager.cc:1367] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.022s user 0.004s sys 0.000s
I20240502 14:14:43.824693 31848 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:43.824867 31854 tablet_bootstrap.cc:654] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.824947 31848 ts_tablet_manager.cc:1367] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.040s user 0.002s sys 0.008s
I20240502 14:14:43.825059 31850 raft_consensus.cc:348] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.825327 31850 raft_consensus.cc:374] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.825429 31850 raft_consensus.cc:729] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.825799 31846 tablet_bootstrap.cc:654] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.825765 31850 consensus_queue.cc:260] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.826012 31848 raft_consensus.cc:348] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.826340 31848 raft_consensus.cc:374] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.826509 31848 raft_consensus.cc:729] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.826850 31848 consensus_queue.cc:260] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.827725 31848 ts_tablet_manager.cc:1398] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 14:14:43.828121 31848 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:43.830359 31852 tablet_bootstrap.cc:654] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.832767 31848 tablet_bootstrap.cc:654] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.833474 31854 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:43.833997 31854 ts_tablet_manager.cc:1367] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.009s
I20240502 14:14:43.834899 31847 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:43.835115 31847 ts_tablet_manager.cc:1367] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.038s user 0.000s sys 0.005s
I20240502 14:14:43.834930 31854 raft_consensus.cc:348] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.835199 31854 raft_consensus.cc:374] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.835364 31854 raft_consensus.cc:729] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.835667 31854 consensus_queue.cc:260] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.836364 31847 raft_consensus.cc:348] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.838719 31847 raft_consensus.cc:374] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.838869 31847 raft_consensus.cc:729] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.839149 31847 consensus_queue.cc:260] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.839550 31846 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:43.839759 31846 ts_tablet_manager.cc:1367] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.003s
I20240502 14:14:43.840080 31847 ts_tablet_manager.cc:1398] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240502 14:14:43.840417 31847 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:43.840615 31846 raft_consensus.cc:348] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.840930 31846 raft_consensus.cc:374] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.841085 31846 raft_consensus.cc:729] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.841364 31846 consensus_queue.cc:260] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.841979 31846 ts_tablet_manager.cc:1398] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:43.842285 31846 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:43.843372 31847 tablet_bootstrap.cc:654] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.844056 31846 tablet_bootstrap.cc:654] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.844558 31850 ts_tablet_manager.cc:1398] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.020s user 0.003s sys 0.000s
I20240502 14:14:43.844864 31850 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:43.846971 31850 tablet_bootstrap.cc:654] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.849217 31846 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:43.849411 31846 ts_tablet_manager.cc:1367] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20240502 14:14:43.850294 31847 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:43.850229 31846 raft_consensus.cc:348] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.850498 31847 ts_tablet_manager.cc:1367] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.003s
I20240502 14:14:43.850610 31846 raft_consensus.cc:374] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.850769 31846 raft_consensus.cc:729] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.851398 31847 raft_consensus.cc:348] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.851740 31847 raft_consensus.cc:374] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.851876 31847 raft_consensus.cc:729] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.852242 31847 consensus_queue.cc:260] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.851109 31846 consensus_queue.cc:260] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.852993 31846 ts_tablet_manager.cc:1398] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:43.853013 31847 ts_tablet_manager.cc:1398] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:14:43.853292 31846 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:43.853338 31847 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:43.855089 31847 tablet_bootstrap.cc:654] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.856938 31846 tablet_bootstrap.cc:654] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.860870 31850 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:43.861202 31850 ts_tablet_manager.cc:1367] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.016s user 0.003s sys 0.003s
I20240502 14:14:43.861761 31852 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:43.861991 31852 ts_tablet_manager.cc:1367] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.046s user 0.006s sys 0.001s
I20240502 14:14:43.862681 31852 raft_consensus.cc:348] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.862951 31852 raft_consensus.cc:374] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.863054 31852 raft_consensus.cc:729] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.863054 31850 raft_consensus.cc:348] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.863711 31850 raft_consensus.cc:374] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.863924 31850 raft_consensus.cc:729] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.864003 31852 consensus_queue.cc:260] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.864384 31850 consensus_queue.cc:260] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.866065 31850 ts_tablet_manager.cc:1398] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240502 14:14:43.866561 31850 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:43.869081 31852 ts_tablet_manager.cc:1398] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240502 14:14:43.870981 31847 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:43.871276 31847 ts_tablet_manager.cc:1367] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.018s user 0.004s sys 0.000s
I20240502 14:14:43.872372 31847 raft_consensus.cc:348] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.872812 31847 raft_consensus.cc:374] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.873226 31852 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:43.873215 31847 raft_consensus.cc:729] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.873589 31847 consensus_queue.cc:260] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.875432 31852 tablet_bootstrap.cc:654] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.876701 31848 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:43.876992 31848 ts_tablet_manager.cc:1367] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.049s user 0.003s sys 0.003s
I20240502 14:14:43.877591 31847 ts_tablet_manager.cc:1398] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 14:14:43.877974 31850 tablet_bootstrap.cc:654] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.878175 31848 raft_consensus.cc:348] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.878610 31848 raft_consensus.cc:374] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.878742 31848 raft_consensus.cc:729] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.879187 31848 consensus_queue.cc:260] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.880124 31848 ts_tablet_manager.cc:1398] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:43.880787 31854 ts_tablet_manager.cc:1398] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.047s user 0.011s sys 0.010s
I20240502 14:14:43.882925 31850 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:43.883165 31850 ts_tablet_manager.cc:1367] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.017s user 0.002s sys 0.002s
I20240502 14:14:43.884550 31850 raft_consensus.cc:348] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.884938 31850 raft_consensus.cc:374] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.885076 31850 raft_consensus.cc:729] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.885500 31850 consensus_queue.cc:260] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.885906 31846 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:43.886119 31846 ts_tablet_manager.cc:1367] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.033s user 0.021s sys 0.007s
I20240502 14:14:43.886288 31850 ts_tablet_manager.cc:1398] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:43.886883 31846 raft_consensus.cc:348] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.887223 31846 raft_consensus.cc:374] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.887379 31846 raft_consensus.cc:729] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.887665 31846 consensus_queue.cc:260] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.888370 31846 ts_tablet_manager.cc:1398] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:43.891157 31852 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:43.891327 31852 ts_tablet_manager.cc:1367] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.018s user 0.000s sys 0.007s
I20240502 14:14:43.892182 31852 raft_consensus.cc:348] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.892444 31852 raft_consensus.cc:374] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.892525 31852 raft_consensus.cc:729] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.892791 31852 consensus_queue.cc:260] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.893229 31852 ts_tablet_manager.cc:1398] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:14:43.893586 31852 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:43.895191 31852 tablet_bootstrap.cc:654] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:43.896082 31841 raft_consensus.cc:482] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:43.896229 31841 raft_consensus.cc:504] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.897039 31841 leader_election.cc:288] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:43.897290 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3183d887deec4e40b88dceda3306c0f1" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:43.897619 30964 raft_consensus.cc:2454] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:43.897660 30702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3183d887deec4e40b88dceda3306c0f1" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:43.897862 30702 raft_consensus.cc:2454] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:43.898207 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3183d887deec4e40b88dceda3306c0f1" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:43.898439 31094 raft_consensus.cc:2454] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:43.898496 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3183d887deec4e40b88dceda3306c0f1" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:43.898793 30833 raft_consensus.cc:2454] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:43.898968 31157 leader_election.cc:302] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 7c1f568f8efc4b2a9dbf38907f02f3e3, d733c51902044956a8cd2b6b31bc9831; no voters:
I20240502 14:14:43.899363 31841 raft_consensus.cc:2781] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:43.899482 31841 raft_consensus.cc:482] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:43.899582 31841 raft_consensus.cc:3037] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.901639 31841 raft_consensus.cc:504] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.902266 31841 leader_election.cc:288] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Requested vote from peers 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:43.902674 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3183d887deec4e40b88dceda3306c0f1" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:43.902878 31094 raft_consensus.cc:3037] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.902835 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3183d887deec4e40b88dceda3306c0f1" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:43.903029 30964 raft_consensus.cc:3037] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.903271 31852 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:43.903206 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3183d887deec4e40b88dceda3306c0f1" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:43.903402 31852 ts_tablet_manager.cc:1367] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.007s
I20240502 14:14:43.903402 30833 raft_consensus.cc:3037] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.904042 31852 raft_consensus.cc:348] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.904287 31852 raft_consensus.cc:374] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:43.904363 31852 raft_consensus.cc:729] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:43.904573 31852 consensus_queue.cc:260] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.905139 31852 ts_tablet_manager.cc:1398] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.002s user 0.001s sys 0.002s
I20240502 14:14:43.905555 30964 raft_consensus.cc:2454] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:43.905957 30702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3183d887deec4e40b88dceda3306c0f1" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:43.906055 30833 raft_consensus.cc:2454] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:43.906134 30702 raft_consensus.cc:3037] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.906586 31094 raft_consensus.cc:2454] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:43.906641 31157 leader_election.cc:302] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3; no voters:
I20240502 14:14:43.907099 31841 raft_consensus.cc:2781] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:43.907236 31841 raft_consensus.cc:686] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Becoming Leader. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
I20240502 14:14:43.907507 31841 consensus_queue.cc:237] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.908193 30702 raft_consensus.cc:2454] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:43.910436 30430 catalog_manager.cc:5484] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: term changed from 0 to 1, leader changed from <none> to 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134). New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:43.927964 31840 raft_consensus.cc:482] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:43.928140 31840 raft_consensus.cc:504] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.929086 31840 leader_election.cc:288] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:14:43.929303 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f7b19e515e497c9a27e8e94cb73079" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:43.929467 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f7b19e515e497c9a27e8e94cb73079" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:43.929577 30833 raft_consensus.cc:2454] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:43.929435 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f7b19e515e497c9a27e8e94cb73079" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:43.929713 31223 raft_consensus.cc:2454] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:43.929811 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f7b19e515e497c9a27e8e94cb73079" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:43.930011 31094 raft_consensus.cc:2454] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:43.930058 30572 raft_consensus.cc:2454] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 0.
I20240502 14:14:43.930176 30900 leader_election.cc:302] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3; no voters:
I20240502 14:14:43.930475 31840 raft_consensus.cc:2781] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:43.930572 31840 raft_consensus.cc:482] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:43.930655 31840 raft_consensus.cc:3037] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.932503 31840 raft_consensus.cc:504] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.933410 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f7b19e515e497c9a27e8e94cb73079" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:43.933410 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f7b19e515e497c9a27e8e94cb73079" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:43.933609 30833 raft_consensus.cc:3037] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.933610 31094 raft_consensus.cc:3037] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.933571 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f7b19e515e497c9a27e8e94cb73079" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:43.933554 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "50f7b19e515e497c9a27e8e94cb73079" candidate_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
I20240502 14:14:43.933739 30572 raft_consensus.cc:3037] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.933751 31223 raft_consensus.cc:3037] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.936313 31094 raft_consensus.cc:2454] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:43.936313 30833 raft_consensus.cc:2454] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:43.936434 31223 raft_consensus.cc:2454] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:43.936434 30572 raft_consensus.cc:2454] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 7c1f568f8efc4b2a9dbf38907f02f3e3 in term 1.
I20240502 14:14:43.936738 30897 leader_election.cc:302] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, ba420f2442d6416bbda95edfff85e849; no voters:
I20240502 14:14:43.936996 31840 leader_election.cc:288] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [CANDIDATE]: Term 1 election: Requested vote from peers 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:14:43.937199 31840 raft_consensus.cc:2781] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:43.937388 31840 raft_consensus.cc:686] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Becoming Leader. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Running, Role: LEADER
I20240502 14:14:43.937793 31840 consensus_queue.cc:237] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } }
I20240502 14:14:43.940873 30430 catalog_manager.cc:5484] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: term changed from 0 to 1, leader changed from <none> to 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:43.965924 31857 raft_consensus.cc:482] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:43.966122 31857 raft_consensus.cc:504] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.967087 31857 leader_election.cc:288] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:43.967411 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c26d346223a498bb96b962fa7dd99aa" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:43.967514 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c26d346223a498bb96b962fa7dd99aa" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:43.967587 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c26d346223a498bb96b962fa7dd99aa" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:43.967675 30572 raft_consensus.cc:2454] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:43.967774 30833 raft_consensus.cc:2454] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:43.967819 30964 raft_consensus.cc:2454] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:43.967765 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c26d346223a498bb96b962fa7dd99aa" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:43.968025 31223 raft_consensus.cc:2454] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:43.968230 31028 leader_election.cc:302] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 7c1f568f8efc4b2a9dbf38907f02f3e3, ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:43.968508 31857 raft_consensus.cc:2781] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:43.968629 31857 raft_consensus.cc:482] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:43.968711 31857 raft_consensus.cc:3037] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.970753 31857 raft_consensus.cc:504] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.971812 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c26d346223a498bb96b962fa7dd99aa" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:43.972059 30833 raft_consensus.cc:3037] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.972044 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c26d346223a498bb96b962fa7dd99aa" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
I20240502 14:14:43.972002 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c26d346223a498bb96b962fa7dd99aa" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:43.972254 31223 raft_consensus.cc:3037] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.972327 30572 raft_consensus.cc:3037] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.974889 30833 raft_consensus.cc:2454] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 1.
I20240502 14:14:43.974989 31223 raft_consensus.cc:2454] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 1.
I20240502 14:14:43.975426 31857 leader_election.cc:288] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 election: Requested vote from peers 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:43.975749 31027 leader_election.cc:302] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, ba420f2442d6416bbda95edfff85e849; no voters:
I20240502 14:14:43.976105 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3c26d346223a498bb96b962fa7dd99aa" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:43.976315 30572 raft_consensus.cc:2454] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 1.
I20240502 14:14:43.976387 30964 raft_consensus.cc:3037] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:43.976609 31857 raft_consensus.cc:2781] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:43.979573 30964 raft_consensus.cc:2454] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 1.
I20240502 14:14:43.981870 31857 raft_consensus.cc:686] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Becoming Leader. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Running, Role: LEADER
I20240502 14:14:43.982484 31857 consensus_queue.cc:237] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:43.987721 30430 catalog_manager.cc:5484] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 reported cstate change: term changed from 0 to 1, leader changed from <none> to ba420f2442d6416bbda95edfff85e849 (127.27.220.133). New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:44.292982 31841 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.353683 31840 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.360849 31840 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:14:44.367147 31874 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:44.367563 31874 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:14:44.372081 31841 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:14:44.379021 31874 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:14:44.379989 31886 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.381943 31888 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.388545 31889 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.391232 31886 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.404870 31857 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.439180 31857 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.483227 31857 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:44.483516 31907 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_3'
INSERT report
rows total: 2000
time total: 368.379 ms
time per row: 0.184189 ms
I20240502 14:14:44.986632 30430 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36920:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:14:44.996670 30813 tablet_service.cc:1467] Processing CreateTablet for tablet 36b27125eebb410e8c37560450778dd5 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:44.997155 30813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36b27125eebb410e8c37560450778dd5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:44.997251 30682 tablet_service.cc:1467] Processing CreateTablet for tablet 36b27125eebb410e8c37560450778dd5 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:44.997732 30682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36b27125eebb410e8c37560450778dd5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.001842 31961 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:45.002082 30552 tablet_service.cc:1467] Processing CreateTablet for tablet 36b27125eebb410e8c37560450778dd5 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.002261 30943 tablet_service.cc:1467] Processing CreateTablet for tablet 36b27125eebb410e8c37560450778dd5 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.002475 30552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36b27125eebb410e8c37560450778dd5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.002717 30943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36b27125eebb410e8c37560450778dd5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.002954 31072 tablet_service.cc:1467] Processing CreateTablet for tablet 36b27125eebb410e8c37560450778dd5 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.003389 31072 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36b27125eebb410e8c37560450778dd5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.003482 31961 tablet_bootstrap.cc:654] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.010560 30553 tablet_service.cc:1467] Processing CreateTablet for tablet bfa09c15ce944ce29fce3bbc1ad4df56 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.010929 30553 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bfa09c15ce944ce29fce3bbc1ad4df56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.010833 31203 tablet_service.cc:1467] Processing CreateTablet for tablet bfa09c15ce944ce29fce3bbc1ad4df56 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.011271 31203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bfa09c15ce944ce29fce3bbc1ad4df56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.019225 30683 tablet_service.cc:1467] Processing CreateTablet for tablet bfa09c15ce944ce29fce3bbc1ad4df56 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.019666 30683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bfa09c15ce944ce29fce3bbc1ad4df56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.020391 30942 tablet_service.cc:1467] Processing CreateTablet for tablet bfa09c15ce944ce29fce3bbc1ad4df56 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.020484 30684 tablet_service.cc:1467] Processing CreateTablet for tablet af7e809edf3d44739814918dc2bff263 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.020903 30942 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bfa09c15ce944ce29fce3bbc1ad4df56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.020973 30684 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af7e809edf3d44739814918dc2bff263. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.023836 30814 tablet_service.cc:1467] Processing CreateTablet for tablet af7e809edf3d44739814918dc2bff263 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.024289 30814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af7e809edf3d44739814918dc2bff263. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.027256 31204 tablet_service.cc:1467] Processing CreateTablet for tablet af7e809edf3d44739814918dc2bff263 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.027747 31204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af7e809edf3d44739814918dc2bff263. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.029440 30813 tablet_service.cc:1467] Processing CreateTablet for tablet bfa09c15ce944ce29fce3bbc1ad4df56 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.030017 30813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bfa09c15ce944ce29fce3bbc1ad4df56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.032526 31964 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:45.034935 31966 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:45.039132 30553 tablet_service.cc:1467] Processing CreateTablet for tablet af7e809edf3d44739814918dc2bff263 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.039577 30553 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af7e809edf3d44739814918dc2bff263. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.041062 31966 tablet_bootstrap.cc:654] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.041622 31964 tablet_bootstrap.cc:654] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.046710 31073 tablet_service.cc:1467] Processing CreateTablet for tablet af7e809edf3d44739814918dc2bff263 (DEFAULT_TABLE table=rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:14:45.047124 31073 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af7e809edf3d44739814918dc2bff263. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:45.054070 31965 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:45.055810 31965 tablet_bootstrap.cc:654] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.057279 31963 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:45.057497 31966 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:45.057832 31966 ts_tablet_manager.cc:1367] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.023s user 0.003s sys 0.001s
I20240502 14:14:45.058735 31966 raft_consensus.cc:348] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.059175 31966 raft_consensus.cc:374] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.059324 31966 raft_consensus.cc:729] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.059628 31966 consensus_queue.cc:260] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.059860 31964 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:45.060063 31964 ts_tablet_manager.cc:1367] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.043s user 0.004s sys 0.002s
I20240502 14:14:45.061898 31967 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:45.062351 31964 raft_consensus.cc:348] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.062666 31964 raft_consensus.cc:374] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.062774 31964 raft_consensus.cc:729] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.063067 31964 consensus_queue.cc:260] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.063859 31964 ts_tablet_manager.cc:1398] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:14:45.064102 31967 tablet_bootstrap.cc:654] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.064185 31964 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849: Bootstrap starting.
I20240502 14:14:45.066440 31966 ts_tablet_manager.cc:1398] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.008s user 0.000s sys 0.002s
I20240502 14:14:45.066771 31966 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:45.067819 31963 tablet_bootstrap.cc:654] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.069320 31965 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:45.069489 31965 ts_tablet_manager.cc:1367] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.000s
I20240502 14:14:45.069950 31966 tablet_bootstrap.cc:654] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.070439 31965 raft_consensus.cc:348] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.070768 31965 raft_consensus.cc:374] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.070863 31965 raft_consensus.cc:729] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.071147 31965 consensus_queue.cc:260] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.071785 31965 ts_tablet_manager.cc:1398] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:14:45.072430 31961 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:45.072630 31961 ts_tablet_manager.cc:1367] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.071s user 0.017s sys 0.030s
I20240502 14:14:45.073423 31964 tablet_bootstrap.cc:654] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.073400 31961 raft_consensus.cc:348] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.073801 31961 raft_consensus.cc:374] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.073900 31961 raft_consensus.cc:729] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.074121 31961 consensus_queue.cc:260] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.074502 31965 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:45.074797 31961 ts_tablet_manager.cc:1398] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:45.075032 31961 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:45.076162 31965 tablet_bootstrap.cc:654] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.076845 31967 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:45.076980 31967 ts_tablet_manager.cc:1367] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.002s
I20240502 14:14:45.077293 31961 tablet_bootstrap.cc:654] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.078105 31967 raft_consensus.cc:348] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.078588 31967 raft_consensus.cc:374] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.078744 31967 raft_consensus.cc:729] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.079108 31967 consensus_queue.cc:260] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.080139 31967 ts_tablet_manager.cc:1398] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:45.080559 31967 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: Bootstrap starting.
I20240502 14:14:45.082684 31967 tablet_bootstrap.cc:654] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.085372 31967 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: No bootstrap required, opened a new log
I20240502 14:14:45.085601 31967 ts_tablet_manager.cc:1367] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 14:14:45.085665 31966 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:45.085837 31965 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:45.085886 31966 ts_tablet_manager.cc:1367] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.019s user 0.002s sys 0.001s
I20240502 14:14:45.086001 31965 ts_tablet_manager.cc:1367] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.000s
I20240502 14:14:45.086581 31967 raft_consensus.cc:348] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.086673 31965 raft_consensus.cc:348] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.086917 31965 raft_consensus.cc:374] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.086949 31967 raft_consensus.cc:374] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.087005 31965 raft_consensus.cc:729] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.086855 31966 raft_consensus.cc:348] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.087110 31967 raft_consensus.cc:729] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.087267 31966 raft_consensus.cc:374] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.087376 31966 raft_consensus.cc:729] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.087280 31965 consensus_queue.cc:260] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.087666 31967 consensus_queue.cc:260] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.087730 31966 consensus_queue.cc:260] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.088510 31967 ts_tablet_manager.cc:1398] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:45.089437 31966 ts_tablet_manager.cc:1398] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:45.089851 31966 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:45.091843 31966 tablet_bootstrap.cc:654] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.092654 31961 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:45.092887 31961 ts_tablet_manager.cc:1367] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.018s user 0.005s sys 0.011s
I20240502 14:14:45.093722 31961 raft_consensus.cc:348] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.093866 31963 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:45.094063 31963 ts_tablet_manager.cc:1367] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.037s user 0.006s sys 0.006s
I20240502 14:14:45.094074 31961 raft_consensus.cc:374] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.094156 31961 raft_consensus.cc:729] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.094441 31961 consensus_queue.cc:260] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.095059 31961 ts_tablet_manager.cc:1398] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:14:45.095191 31963 raft_consensus.cc:348] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.095371 31961 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:45.095518 31963 raft_consensus.cc:374] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.095657 31963 raft_consensus.cc:729] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.095957 31963 consensus_queue.cc:260] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.097347 31961 tablet_bootstrap.cc:654] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.100970 31963 ts_tablet_manager.cc:1398] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 14:14:45.101349 31963 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Bootstrap starting.
I20240502 14:14:45.101770 31966 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831: No bootstrap required, opened a new log
I20240502 14:14:45.101927 31966 ts_tablet_manager.cc:1367] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.009s
I20240502 14:14:45.102694 31966 raft_consensus.cc:348] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.102972 31966 raft_consensus.cc:374] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.103066 31966 raft_consensus.cc:729] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.103327 31966 consensus_queue.cc:260] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.104552 31966 ts_tablet_manager.cc:1398] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:45.105930 31961 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494: No bootstrap required, opened a new log
I20240502 14:14:45.106104 31961 ts_tablet_manager.cc:1367] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 0.011s user 0.005s sys 0.004s
I20240502 14:14:45.106248 31963 tablet_bootstrap.cc:654] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.107734 31961 raft_consensus.cc:348] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.108060 31961 raft_consensus.cc:374] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.108152 31961 raft_consensus.cc:729] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.108624 31961 consensus_queue.cc:260] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.109290 31961 ts_tablet_manager.cc:1398] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:14:45.110075 31963 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3: No bootstrap required, opened a new log
I20240502 14:14:45.110239 31963 ts_tablet_manager.cc:1367] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.000s
I20240502 14:14:45.110778 31963 raft_consensus.cc:348] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.111013 31963 raft_consensus.cc:374] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.111089 31963 raft_consensus.cc:729] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 7c1f568f8efc4b2a9dbf38907f02f3e3, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.111330 31963 consensus_queue.cc:260] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.112048 31963 ts_tablet_manager.cc:1398] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:45.113575 31965 ts_tablet_manager.cc:1398] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.027s user 0.002s sys 0.000s
I20240502 14:14:45.113938 31965 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f: Bootstrap starting.
I20240502 14:14:45.115197 31964 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849: No bootstrap required, opened a new log
I20240502 14:14:45.115410 31964 ts_tablet_manager.cc:1367] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849: Time spent bootstrapping tablet: real 0.051s user 0.002s sys 0.002s
I20240502 14:14:45.115698 31965 tablet_bootstrap.cc:654] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f: Neither blocks nor log segments found. Creating new log.
I20240502 14:14:45.118523 31964 raft_consensus.cc:348] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.118804 31964 raft_consensus.cc:374] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.118870 31964 raft_consensus.cc:729] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.119138 31964 consensus_queue.cc:260] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.120376 31964 ts_tablet_manager.cc:1398] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
I20240502 14:14:45.121601 31965 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f: No bootstrap required, opened a new log
I20240502 14:14:45.122359 31965 ts_tablet_manager.cc:1367] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f: Time spent bootstrapping tablet: real 0.009s user 0.007s sys 0.000s
I20240502 14:14:45.123023 31965 raft_consensus.cc:348] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.123258 31965 raft_consensus.cc:374] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:14:45.123335 31965 raft_consensus.cc:729] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Initialized, Role: FOLLOWER
I20240502 14:14:45.123551 31965 consensus_queue.cc:260] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.124199 31965 ts_tablet_manager.cc:1398] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:14:45.187551 31872 raft_consensus.cc:482] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:45.187731 31872 raft_consensus.cc:504] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.188648 31872 leader_election.cc:288] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:14:45.189023 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36b27125eebb410e8c37560450778dd5" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:45.188999 30702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36b27125eebb410e8c37560450778dd5" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:45.189244 30833 raft_consensus.cc:2454] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:45.189281 30702 raft_consensus.cc:2454] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:45.189180 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36b27125eebb410e8c37560450778dd5" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:45.189280 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36b27125eebb410e8c37560450778dd5" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:45.189491 30572 raft_consensus.cc:2454] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:45.189481 30964 raft_consensus.cc:2454] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 0.
I20240502 14:14:45.189828 31027 leader_election.cc:302] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, ba420f2442d6416bbda95edfff85e849, d733c51902044956a8cd2b6b31bc9831; no voters:
I20240502 14:14:45.190115 31872 raft_consensus.cc:2781] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:45.190222 31872 raft_consensus.cc:482] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:45.190325 31872 raft_consensus.cc:3037] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.192600 31872 raft_consensus.cc:504] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.193558 31872 leader_election.cc:288] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 election: Requested vote from peers 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:14:45.193759 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36b27125eebb410e8c37560450778dd5" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:45.193971 30964 raft_consensus.cc:3037] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.193908 30702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36b27125eebb410e8c37560450778dd5" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:45.194128 30702 raft_consensus.cc:3037] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.194177 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36b27125eebb410e8c37560450778dd5" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:45.194185 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36b27125eebb410e8c37560450778dd5" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:45.194378 30572 raft_consensus.cc:3037] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.194380 30833 raft_consensus.cc:3037] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.196925 30702 raft_consensus.cc:2454] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 1.
I20240502 14:14:45.197151 30833 raft_consensus.cc:2454] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 1.
I20240502 14:14:45.196677 30964 raft_consensus.cc:2454] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 1.
I20240502 14:14:45.197571 31027 leader_election.cc:302] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 155f26c73dc84441a58417c09f577494, ba420f2442d6416bbda95edfff85e849, d733c51902044956a8cd2b6b31bc9831; no voters:
I20240502 14:14:45.197085 30572 raft_consensus.cc:2454] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ba420f2442d6416bbda95edfff85e849 in term 1.
I20240502 14:14:45.197933 31872 raft_consensus.cc:2781] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:45.198077 31872 raft_consensus.cc:686] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Becoming Leader. State: Replica: ba420f2442d6416bbda95edfff85e849, State: Running, Role: LEADER
I20240502 14:14:45.198402 31872 consensus_queue.cc:237] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.201532 30429 catalog_manager.cc:5484] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 reported cstate change: term changed from 0 to 1, leader changed from <none> to ba420f2442d6416bbda95edfff85e849 (127.27.220.133). New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } }
I20240502 14:14:45.240821 31701 raft_consensus.cc:482] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:45.241044 31701 raft_consensus.cc:504] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.242079 31701 leader_election.cc:288] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:14:45.242285 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f" is_pre_election: true
I20240502 14:14:45.242542 30572 raft_consensus.cc:2454] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:45.242552 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:45.242743 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" is_pre_election: true
I20240502 14:14:45.242615 30702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:45.242863 30833 raft_consensus.cc:2454] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:45.242957 30964 raft_consensus.cc:2454] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:45.243083 30702 raft_consensus.cc:2454] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 0.
I20240502 14:14:45.243320 31157 leader_election.cc:302] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:45.243621 31701 raft_consensus.cc:2781] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:45.243746 31701 raft_consensus.cc:482] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:45.243852 31701 raft_consensus.cc:3037] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.245860 31701 raft_consensus.cc:504] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.246685 31701 leader_election.cc:288] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Requested vote from peers fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:14:45.246963 30702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:45.247053 30572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "fd3399b1bfe644159a833ede6726c25f"
I20240502 14:14:45.247040 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:45.247097 30964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:14:45.247246 30572 raft_consensus.cc:3037] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.247277 30964 raft_consensus.cc:3037] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.247279 30833 raft_consensus.cc:3037] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.247289 30702 raft_consensus.cc:3037] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.249992 30572 raft_consensus.cc:2454] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:45.250077 30964 raft_consensus.cc:2454] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:45.250077 30833 raft_consensus.cc:2454] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:45.250458 30702 raft_consensus.cc:2454] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:14:45.250591 31157 leader_election.cc:302] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:45.250998 31701 raft_consensus.cc:2781] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:45.251199 31701 raft_consensus.cc:686] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Becoming Leader. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
I20240502 14:14:45.251519 31701 consensus_queue.cc:237] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } }
I20240502 14:14:45.254963 30429 catalog_manager.cc:5484] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: term changed from 0 to 1, leader changed from <none> to 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134). New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:45.320448 31910 raft_consensus.cc:482] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:14:45.320676 31910 raft_consensus.cc:504] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.321868 31910 leader_election.cc:288] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:45.322281 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:14:45.322537 31223 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 0.
I20240502 14:14:45.322788 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:14:45.323118 31094 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 0.
I20240502 14:14:45.323714 30508 leader_election.cc:302] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:45.324095 31910 raft_consensus.cc:2781] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:14:45.324113 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494" is_pre_election: true
I20240502 14:14:45.324285 31910 raft_consensus.cc:482] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:14:45.324350 30833 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 0.
I20240502 14:14:45.324409 31910 raft_consensus.cc:3037] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.327466 31910 raft_consensus.cc:504] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.328398 31910 leader_election.cc:288] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [CANDIDATE]: Term 1 election: Requested vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:45.328595 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
I20240502 14:14:45.328703 30833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:14:45.328797 31223 raft_consensus.cc:3037] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.328918 30833 raft_consensus.cc:3037] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.329314 31094 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:14:45.329571 31094 raft_consensus.cc:3037] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.331912 30833 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 1.
I20240502 14:14:45.331912 31223 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 1.
I20240502 14:14:45.332394 30507 leader_election.cc:302] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 155f26c73dc84441a58417c09f577494, fd3399b1bfe644159a833ede6726c25f; no voters:
I20240502 14:14:45.332463 31094 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 1.
I20240502 14:14:45.332551 30703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:14:45.332711 31910 raft_consensus.cc:2781] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:14:45.332800 30703 raft_consensus.cc:3037] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:14:45.332551 30702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "fd3399b1bfe644159a833ede6726c25f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d733c51902044956a8cd2b6b31bc9831" is_pre_election: true
I20240502 14:14:45.334257 31910 raft_consensus.cc:686] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Becoming Leader. State: Replica: fd3399b1bfe644159a833ede6726c25f, State: Running, Role: LEADER
I20240502 14:14:45.334604 31910 consensus_queue.cc:237] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } }
I20240502 14:14:45.335775 30703 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fd3399b1bfe644159a833ede6726c25f in term 1.
I20240502 14:14:45.338924 30429 catalog_manager.cc:5484] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f reported cstate change: term changed from 0 to 1, leader changed from <none> to fd3399b1bfe644159a833ede6726c25f (127.27.220.129). New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:45.634184 31872 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.637682 31954 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.708173 31954 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.717129 31872 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.727635 31910 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.734119 31701 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.749382 31910 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.752276 31701 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:14:45.773195 31990 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:45.773682 31990 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:45.787662 31990 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:14:45.791349 31872 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.794303 31988 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.795977 31988 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:14:45.813673 31872 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_4'
INSERT report
rows total: 2000
time total: 310.62 ms
time per row: 0.15531 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 14:14:47.215731 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.135:33091
--local_ip_for_outbound_sockets=127.27.220.135
--tserver_master_addrs=127.27.220.190:33157
--webserver_port=37645
--webserver_interface=127.27.220.135
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:14:47.558198 32074 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:14:47.558497 32074 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:14:47.558598 32074 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:14:47.567126 32074 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:14:47.567276 32074 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:14:47.567430 32074 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.135
I20240502 14:14:47.576719 32074 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.135:33091
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.27.220.135
--webserver_port=37645
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.135
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:14:47.578171 32074 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:14:47.580004 32074 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:14:47.593858 32081 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:47.594421 32082 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:14:47.595131 32084 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:14:47.595966 32074 server_base.cc:1034] running on GCE node
I20240502 14:14:47.596662 32074 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:14:47.597874 32074 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:14:47.599076 32074 hybrid_clock.cc:648] HybridClock initialized: now 1714659287599019 us; error 48 us; skew 500 ppm
I20240502 14:14:47.599403 32074 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:14:47.601648 32074 webserver.cc:460] Webserver started at http://127.27.220.135:37645/ using document root <none> and password file <none>
I20240502 14:14:47.602373 32074 fs_manager.cc:359] Metadata directory not provided
I20240502 14:14:47.602499 32074 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:14:47.607978 32074 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 14:14:47.610706 32090 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:47.611948 32074 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:14:47.612094 32074 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "3eafb09239024de3bc0eea8d700f7668"
format_stamp: "Formatted at 2024-05-02 14:14:34 on dist-test-slave-j72s"
I20240502 14:14:47.612880 32074 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:14:47.637281 32074 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:14:47.638255 32074 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:14:47.638552 32074 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:14:47.639567 32074 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:14:47.641098 32074 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:14:47.641180 32074 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:47.641306 32074 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:14:47.641362 32074 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:14:47.675207 32074 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.135:33091
I20240502 14:14:47.675312 32202 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.135:33091 every 8 connection(s)
I20240502 14:14:47.676801 32074 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 14:14:47.684942 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 32074
I20240502 14:14:47.685241 28530 test_util.cc:268] Using random seed: 968939618
I20240502 14:14:47.690213 32203 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:14:47.690595 32203 heartbeater.cc:464] Registering TS with master...
I20240502 14:14:47.691717 32203 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:14:47.693361 30429 ts_manager.cc:194] Re-registered known tserver with Master: 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091)
I20240502 14:14:47.694497 30429 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.135:56585
I20240502 14:14:47.702206 28530 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_0 already exists
I20240502 14:14:47.702695 28530 test_util.cc:268] Using random seed: 968957068
I20240502 14:14:47.719659 28530 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_1 already exists
I20240502 14:14:47.720098 28530 test_util.cc:268] Using random seed: 968974463
I20240502 14:14:47.770681 28530 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_2 already exists
I20240502 14:14:47.815621 28530 test_util.cc:268] Using random seed: 969069955
W20240502 14:14:47.818212 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 43617.6us
I20240502 14:14:47.872471 28530 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_3 already exists
I20240502 14:14:47.886260 28530 test_util.cc:268] Using random seed: 969140617
W20240502 14:14:47.934691 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 45648.9us
I20240502 14:14:47.969620 28530 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_4 already exists
W20240502 14:14:48.693182 30768 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 92591.4us
I20240502 14:14:48.697315 32203 heartbeater.cc:503] Master 127.27.220.190:33157 was elected leader, sending a full tablet report...
W20240502 14:14:49.277750 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 67802.8us
W20240502 14:14:49.293761 30508 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 123928us
W20240502 14:14:49.356842 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 72220.4us
W20240502 14:14:50.340893 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 101207us
W20240502 14:14:50.732144 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 45620.8us
W20240502 14:14:50.805539 30508 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 64462.8us
W20240502 14:14:50.899103 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 100356us
W20240502 14:14:50.981976 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 68449.3us
W20240502 14:14:51.256212 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:41511, user_credentials={real_user=slave}} blocked reactor thread for 69982us
I20240502 14:14:51.518705 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:14:51.562621 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:51.711100 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:14:51.737681 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:51.764685 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20240502 14:14:51.775274 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 53212.1us
W20240502 14:14:51.881884 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 51173.2us
I20240502 14:14:51.910905 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:14:52.007517 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20240502 14:14:52.316440 31027 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 57348.1us
W20240502 14:14:52.531072 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:41511, user_credentials={real_user=slave}} blocked reactor thread for 43989.3us
W20240502 14:14:52.971583 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 131356us
W20240502 14:14:53.143224 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 71497.4us
I20240502 14:14:54.078075 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:14:54.108896 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:14:54.134443 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:14:54.261029 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:14:54.265889 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:14:54.357656 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 14:14:54.447721 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 56291.1us
I20240502 14:14:54.459049 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:14:55.462232 30832 consensus_queue.cc:237] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 40, Committed index: 40, Last appended: 1.40, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:55.467393 31092 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 40. Preceding OpId from leader: term: 1 index: 41. (index mismatch)
I20240502 14:14:55.468102 32362 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 40, Time since last communication: 0.000s
W20240502 14:14:55.469564 30898 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 71012us
I20240502 14:14:55.471443 30964 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 40. Preceding OpId from leader: term: 1 index: 41. (index mismatch)
I20240502 14:14:55.472043 32362 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 40, Time since last communication: 0.000s
I20240502 14:14:55.478405 32578 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 LEADER]: Committing config change with OpId 1.41: config changed from index -1 to 41, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:55.481990 31222 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 40. Preceding OpId from leader: term: 1 index: 41. (index mismatch)
I20240502 14:14:55.489462 30958 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.41: config changed from index -1 to 41, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:55.501681 31092 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.41: config changed from index -1 to 41, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:55.510196 30429 catalog_manager.cc:5484] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index -1 to 41, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "155f26c73dc84441a58417c09f577494" committed_config { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:55.519021 32362 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 40, Time since last communication: 0.000s
I20240502 14:14:55.520256 30703 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 40. Preceding OpId from leader: term: 1 index: 41. (index mismatch)
I20240502 14:14:55.528170 32362 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 41, Last known committed idx: 40, Time since last communication: 0.000s
I20240502 14:14:55.530634 30703 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.41: config changed from index -1 to 41, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
W20240502 14:14:55.535580 30767 consensus_peers.cc:485] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0f906c7d775540e6ba76cf2ff3700a67. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:55.648798 31222 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.41: config changed from index -1 to 41, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
W20240502 14:14:55.813637 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 68956.7us
W20240502 14:14:55.855178 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 43648us
W20240502 14:14:55.890074 30406 kernel_stack_watchdog.cc:198] Thread 32606 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/log.cc:407 for 246ms:
Kernel stack:
(could not read kernel stack)
User stack:
@ 0x7f09a3f61980 (unknown) at ??:0
@ 0x90ef77 __sanitizer::internal_sched_yield() at ??:?
@ 0x87462d __interceptor_pthread_create at ??:?
@ 0x7f09a0bcfd11 kudu::Thread::StartThread() at ??:0
@ 0xa4d187 kudu::Thread::Create() at ??:?
@ 0x7f09a0bf07ab kudu::ThreadPool::CreateThread() at ??:0
@ 0x7f09a0bec3ad kudu::ThreadPool::DoSubmit() at ??:0
@ 0x7f09a0bea84f kudu::ThreadPoolToken::Submit() at ??:0
@ 0x7f09ac34d1de kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange() at ??:0
@ 0x7f09ac33b9cc kudu::consensus::PeerMessageQueue::ResponseFromPeer() at ??:0
@ 0x7f09ac3397a4 kudu::consensus::PeerMessageQueue::LocalPeerAppendFinished() at ??:0
@ 0x7f09ac3514da kudu::consensus::PeerMessageQueue::AppendOperations()::$_1::operator()() at ??:0
@ 0x7f09ac351242 std::_Function_handler<>::_M_invoke() at ??:0
@ 0x7f09b40fd17e std::function<>::operator()() at ??:0
@ 0x7f09ac39e57e kudu::consensus::LogCache::LogCallback() at ??:0
@ 0x7f09ac3a335f kudu::consensus::LogCache::AppendOperations()::$_0::operator()() at ??:0
I20240502 14:14:55.908190 31223 consensus_queue.cc:237] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 56, Committed index: 56, Last appended: 1.56, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } }
I20240502 14:14:55.912072 30572 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:55.912662 32279 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 14:14:55.917309 31091 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:55.918469 32545 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 14:14:55.922786 32589 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } } }
I20240502 14:14:55.923641 31093 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } } }
I20240502 14:14:55.937254 30571 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } } }
I20240502 14:14:55.945439 30428 catalog_manager.cc:5484] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index -1 to 57, NON_VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } } }
I20240502 14:14:55.955678 30958 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
W20240502 14:14:55.963778 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 50368.7us
I20240502 14:14:55.974048 32276 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 14:14:55.982836 30961 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } } }
I20240502 14:14:56.015372 30703 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
W20240502 14:14:56.063102 31157 consensus_peers.cc:485] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d4aa1eb08d4b450f9c5be499d8c6bbc7. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:56.068328 32276 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 14:14:56.081282 31092 consensus_queue.cc:237] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 54, Committed index: 54, Last appended: 1.54, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
W20240502 14:14:56.085340 30508 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 95938.6us
I20240502 14:14:56.090741 30961 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 54. Preceding OpId from leader: term: 1 index: 55. (index mismatch)
I20240502 14:14:56.091341 32076 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 55, Last known committed idx: 54, Time since last communication: 0.000s
I20240502 14:14:56.092216 30571 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 54. Preceding OpId from leader: term: 1 index: 55. (index mismatch)
I20240502 14:14:56.093828 32635 ts_tablet_manager.cc:902] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:14:56.096293 32076 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 55, Last known committed idx: 54, Time since last communication: 0.000s
I20240502 14:14:56.097174 30699 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 54. Preceding OpId from leader: term: 1 index: 55. (index mismatch)
I20240502 14:14:56.100564 32076 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 55, Last known committed idx: 54, Time since last communication: 0.000s
I20240502 14:14:56.102838 32076 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.55: config changed from index -1 to 55, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.110033 30573 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.55: config changed from index -1 to 55, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.117482 30428 catalog_manager.cc:5484] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index -1 to 55, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.118932 30699 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.55: config changed from index -1 to 55, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.131135 30829 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 54. Preceding OpId from leader: term: 1 index: 55. (index mismatch)
I20240502 14:14:56.133746 32434 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 55, Last known committed idx: 54, Time since last communication: 0.000s
I20240502 14:14:56.135464 30828 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.55: config changed from index -1 to 55, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.141717 30961 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.55: config changed from index -1 to 55, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
W20240502 14:14:56.153177 31027 consensus_peers.cc:485] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 36b27125eebb410e8c37560450778dd5. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:56.200709 32635 tablet_copy_client.cc:308] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:34975
I20240502 14:14:56.229509 30699 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } } }
I20240502 14:14:56.280453 31089 consensus_queue.cc:237] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 56, Committed index: 56, Last appended: 1.56, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } }
I20240502 14:14:56.283262 30961 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:56.283340 31223 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:56.283787 32076 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 14:14:56.284215 32076 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 14:14:56.287245 30574 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:56.287796 32355 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 55, Time since last communication: 0.000s
I20240502 14:14:56.291870 32355 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } } }
I20240502 14:14:56.296489 31223 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } } }
I20240502 14:14:56.312166 30574 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } } }
I20240502 14:14:56.314680 30961 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } } }
I20240502 14:14:56.321152 30428 catalog_manager.cc:5484] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index -1 to 57, NON_VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) added. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } } }
I20240502 14:14:56.389962 31223 consensus_queue.cc:237] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 68, Committed index: 68, Last appended: 1.69, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:56.395578 30961 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 14:14:56.396154 32276 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 68, Time since last communication: 0.000s
I20240502 14:14:56.411350 30828 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 55. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:56.420756 31089 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 14:14:56.443928 32276 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 68, Time since last communication: 0.000s
I20240502 14:14:56.447252 32373 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 54, Time since last communication: 0.000s
W20240502 14:14:56.463555 31027 consensus_peers.cc:485] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3c26d346223a498bb96b962fa7dd99aa. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:56.466284 30833 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index -1 to 57, NON_VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) added. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } } }
I20240502 14:14:56.474468 30696 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 14:14:56.475270 30828 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 14:14:56.478523 32454 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 68, Time since last communication: 0.000s
I20240502 14:14:56.545694 32279 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.70: config changed from index -1 to 70, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.582832 32665 ts_tablet_manager.cc:902] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:56.583194 30961 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index -1 to 70, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.590107 30428 catalog_manager.cc:5484] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index -1 to 70, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.596324 31091 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index -1 to 70, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.611737 30854 tablet_copy_service.cc:140] P 155f26c73dc84441a58417c09f577494: Received BeginTabletCopySession request for tablet 0f906c7d775540e6ba76cf2ff3700a67 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:44687)
I20240502 14:14:56.612133 30854 tablet_copy_service.cc:161] P 155f26c73dc84441a58417c09f577494: Beginning new tablet copy session on tablet 0f906c7d775540e6ba76cf2ff3700a67 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:44687: session id = 3eafb09239024de3bc0eea8d700f7668-0f906c7d775540e6ba76cf2ff3700a67
I20240502 14:14:56.616377 30854 tablet_copy_source_session.cc:205] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:14:56.622841 32635 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f906c7d775540e6ba76cf2ff3700a67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:56.623956 32662 ts_tablet_manager.cc:902] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:56.638167 32665 tablet_copy_client.cc:308] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:34805
I20240502 14:14:56.646386 31114 tablet_copy_service.cc:140] P ba420f2442d6416bbda95edfff85e849: Received BeginTabletCopySession request for tablet 36b27125eebb410e8c37560450778dd5 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:46789)
I20240502 14:14:56.646593 31114 tablet_copy_service.cc:161] P ba420f2442d6416bbda95edfff85e849: Beginning new tablet copy session on tablet 36b27125eebb410e8c37560450778dd5 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:46789: session id = 3eafb09239024de3bc0eea8d700f7668-36b27125eebb410e8c37560450778dd5
I20240502 14:14:56.651000 31114 tablet_copy_source_session.cc:205] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:14:56.653137 32665 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36b27125eebb410e8c37560450778dd5. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:56.661612 32635 tablet_copy_client.cc:783] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:14:56.662016 32635 tablet_copy_client.cc:647] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:14:56.667130 32635 tablet_copy_client.cc:523] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
W20240502 14:14:56.669867 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 190932us
I20240502 14:14:56.670449 32454 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 68, Time since last communication: 0.000s
I20240502 14:14:56.681206 32665 tablet_copy_client.cc:783] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:14:56.681484 32665 tablet_copy_client.cc:647] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
W20240502 14:14:56.688100 31157 consensus_peers.cc:485] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3183d887deec4e40b88dceda3306c0f1. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:56.688446 32665 tablet_copy_client.cc:523] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:14:56.693451 30827 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index -1 to 70, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.697674 32665 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:14:56.703552 30703 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index -1 to 70, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:56.710273 32635 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:14:56.789680 32665 log.cc:824] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: Log is configured to *not* fsync() on all Append() calls
I20240502 14:14:56.955808 32677 ts_tablet_manager.cc:902] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: Initiating tablet copy from peer ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:14:57.026530 32695 ts_tablet_manager.cc:902] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
W20240502 14:14:57.040619 30508 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 69281.8us
I20240502 14:14:57.111009 32662 tablet_copy_client.cc:308] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:14:57.158856 31244 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 from peer 155f26c73dc84441a58417c09f577494 ({username='slave'} at 127.27.220.131:33683)
I20240502 14:14:57.159159 31244 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 from peer 155f26c73dc84441a58417c09f577494 at {username='slave'} at 127.27.220.131:33683: session id = 155f26c73dc84441a58417c09f577494-d4aa1eb08d4b450f9c5be499d8c6bbc7
I20240502 14:14:57.166973 32677 tablet_copy_client.cc:308] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:34805
I20240502 14:14:57.170181 32695 tablet_copy_client.cc:308] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:14:57.170523 31114 tablet_copy_service.cc:140] P ba420f2442d6416bbda95edfff85e849: Received BeginTabletCopySession request for tablet 3c26d346223a498bb96b962fa7dd99aa from peer d733c51902044956a8cd2b6b31bc9831 ({username='slave'} at 127.27.220.130:36045)
I20240502 14:14:57.170701 31114 tablet_copy_service.cc:161] P ba420f2442d6416bbda95edfff85e849: Beginning new tablet copy session on tablet 3c26d346223a498bb96b962fa7dd99aa from peer d733c51902044956a8cd2b6b31bc9831 at {username='slave'} at 127.27.220.130:36045: session id = d733c51902044956a8cd2b6b31bc9831-3c26d346223a498bb96b962fa7dd99aa
I20240502 14:14:57.175266 31223 consensus_queue.cc:237] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 89, Committed index: 89, Last appended: 1.90, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:57.178198 31114 tablet_copy_source_session.cc:205] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:14:57.179930 30571 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 90. Preceding OpId from leader: term: 1 index: 91. (index mismatch)
I20240502 14:14:57.180202 31092 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 90. Preceding OpId from leader: term: 1 index: 91. (index mismatch)
I20240502 14:14:57.180778 32454 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 91, Last known committed idx: 89, Time since last communication: 0.000s
I20240502 14:14:57.503508 32635 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=44 overwritten=0 applied=44 ignored=0} inserts{seen=675 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:14:57.561776 32635 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:14:57.562325 32635 ts_tablet_manager.cc:1367] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.852s user 0.097s sys 0.010s
I20240502 14:14:57.267926 30833 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 89. Preceding OpId from leader: term: 1 index: 91. (index mismatch)
I20240502 14:14:57.182827 32707 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 91, Last known committed idx: 89, Time since last communication: 0.000s
I20240502 14:14:57.182032 32677 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c26d346223a498bb96b962fa7dd99aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:57.268420 30696 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 89. Preceding OpId from leader: term: 1 index: 91. (index mismatch)
W20240502 14:14:57.568285 31157 consensus_peers.cc:485] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8b1b5e519c314a83bf1e39e011440fa6. This is attempt 1: this message will repeat every 5th retry.
W20240502 14:14:57.568418 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 195796us
I20240502 14:14:57.185948 31244 tablet_copy_source_session.cc:205] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:14:57.571084 32387 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 91, Last known committed idx: 88, Time since last communication: 0.000s
I20240502 14:14:57.571851 31243 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet 3183d887deec4e40b88dceda3306c0f1 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:57521)
I20240502 14:14:57.572127 31243 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet 3183d887deec4e40b88dceda3306c0f1 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:57521: session id = 3eafb09239024de3bc0eea8d700f7668-3183d887deec4e40b88dceda3306c0f1
I20240502 14:14:57.577170 32662 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:57.576808 32707 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.91: config changed from index -1 to 91, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:57.582442 31092 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.91: config changed from index -1 to 91, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:57.592960 30574 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.91: config changed from index -1 to 91, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:57.602854 30430 catalog_manager.cc:5484] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index -1 to 91, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:57.604176 31243 tablet_copy_source_session.cc:205] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:14:57.607578 32662 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.StartTabletCopy from 127.27.220.134:46713 (request call id 559) took 1133 ms. Trace:
I20240502 14:14:57.607729 32662 rpcz_store.cc:276] 0502 14:14:56.474032 (+ 0us) service_pool.cc:168] Inserting onto call queue
0502 14:14:56.493638 (+ 19606us) service_pool.cc:225] Handling call
0502 14:14:56.624019 (+130381us) ts_tablet_manager.cc:903] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
0502 14:14:57.576323 (+952304us) tablet_copy_client.cc:353] Tablet copy session begun
0502 14:14:57.604586 (+ 28263us) tablet_metadata.cc:641] Metadata flushed
0502 14:14:57.604725 (+ 139us) tablet_copy_client.cc:482] Wrote new tablet metadata
0502 14:14:57.607222 (+ 2497us) tablet_copy_client.cc:403] Wrote new consensus metadata
0502 14:14:57.607412 (+ 190us) tablet_replica.cc:205] Creating consensus instance
0502 14:14:57.607502 (+ 90us) ts_tablet_manager.cc:958] Replica registered.
0502 14:14:57.607537 (+ 35us) inbound_call.cc:173] Queueing success response
Metrics: {"dns_us":103,"tablet-copy.queue_time_us":129661,"thread_start_us":616701,"threads_started":5}
I20240502 14:14:57.608824 32662 tablet_copy_client.cc:783] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: tablet copy: Starting download of 0 data blocks...
I20240502 14:14:57.611511 32635 raft_consensus.cc:348] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:57.614851 32677 tablet_copy_client.cc:783] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: tablet copy: Starting download of 0 data blocks...
I20240502 14:14:57.615226 32677 tablet_copy_client.cc:647] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: tablet copy: Starting download of 1 WAL segments...
I20240502 14:14:57.617861 32677 tablet_copy_client.cc:523] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:14:57.624651 32635 raft_consensus.cc:729] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:14:57.626003 32635 consensus_queue.cc:260] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 44, Last appended: 1.44, Last appended by leader: 44, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 41 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:57.626700 32695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3183d887deec4e40b88dceda3306c0f1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:57.634137 32662 tablet_copy_client.cc:647] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: tablet copy: Starting download of 1 WAL segments...
I20240502 14:14:57.640568 32677 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: Bootstrap starting.
I20240502 14:14:57.647948 32665 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=59 overwritten=0 applied=59 ignored=0} inserts{seen=744 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:14:57.665324 32695 tablet_copy_client.cc:783] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:14:57.666814 32695 tablet_copy_client.cc:647] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:14:57.667719 32736 ts_tablet_manager.cc:902] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:14:57.669467 32665 tablet_bootstrap.cc:492] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:14:57.669823 32665 ts_tablet_manager.cc:1367] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.972s user 0.120s sys 0.021s
I20240502 14:14:57.674196 32665 raft_consensus.cc:348] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:57.680701 32279 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 91, Last known committed idx: 89, Time since last communication: 0.000s
I20240502 14:14:57.685931 30697 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.91: config changed from index -1 to 91, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:57.678891 32635 ts_tablet_manager.cc:1398] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.116s user 0.006s sys 0.011s
I20240502 14:14:57.698431 32665 raft_consensus.cc:729] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:14:57.699347 30854 tablet_copy_service.cc:342] P 155f26c73dc84441a58417c09f577494: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-0f906c7d775540e6ba76cf2ff3700a67 received from {username='slave'} at 127.27.220.135:44687
I20240502 14:14:57.699522 30854 tablet_copy_service.cc:433] P 155f26c73dc84441a58417c09f577494: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-0f906c7d775540e6ba76cf2ff3700a67 on tablet 0f906c7d775540e6ba76cf2ff3700a67 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:14:57.700237 32665 consensus_queue.cc:260] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 59, Last appended: 1.59, Last appended by leader: 59, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 55 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:57.699965 32736 tablet_copy_client.cc:308] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:14:57.702761 32665 ts_tablet_manager.cc:1398] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.029s user 0.000s sys 0.002s
I20240502 14:14:57.704613 31114 tablet_copy_service.cc:342] P ba420f2442d6416bbda95edfff85e849: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-36b27125eebb410e8c37560450778dd5 received from {username='slave'} at 127.27.220.135:46789
I20240502 14:14:57.704777 31114 tablet_copy_service.cc:433] P ba420f2442d6416bbda95edfff85e849: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-36b27125eebb410e8c37560450778dd5 on tablet 36b27125eebb410e8c37560450778dd5 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:14:57.732268 31244 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet 8b1b5e519c314a83bf1e39e011440fa6 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:57521)
I20240502 14:14:57.732515 31244 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet 8b1b5e519c314a83bf1e39e011440fa6 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:57521: session id = 3eafb09239024de3bc0eea8d700f7668-8b1b5e519c314a83bf1e39e011440fa6
I20240502 14:14:57.734941 32158 raft_consensus.cc:1203] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.40->[1.41-1.52] Dedup: 1.44->[1.45-1.52]
I20240502 14:14:57.735873 30833 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.91: config changed from index -1 to 91, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:57.738119 31244 tablet_copy_source_session.cc:205] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:14:57.762085 32736 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b1b5e519c314a83bf1e39e011440fa6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:14:57.762256 32662 tablet_copy_client.cc:523] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:14:57.767268 32662 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: Bootstrap starting.
I20240502 14:14:57.772173 32695 tablet_copy_client.cc:523] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:14:57.776026 32695 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:14:57.786899 32736 tablet_copy_client.cc:783] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:14:57.787297 32736 tablet_copy_client.cc:647] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
W20240502 14:14:58.015646 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 181688us
W20240502 14:14:58.104655 30430 thread.cc:636] prepare [worker] (thread pool) Time spent creating pthread: real 0.501s user 0.033s sys 0.090s
W20240502 14:14:58.107518 30430 thread.cc:602] prepare [worker] (thread pool) Time spent starting thread: real 0.504s user 0.033s sys 0.090s
I20240502 14:14:58.118825 32157 raft_consensus.cc:1203] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.54->[1.55-1.66] Dedup: 1.59->[1.60-1.66]
I20240502 14:14:58.140417 32695 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=77 overwritten=0 applied=77 ignored=0} inserts{seen=708 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:14:58.140893 32695 tablet_bootstrap.cc:492] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:14:58.141359 32695 ts_tablet_manager.cc:1367] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.365s user 0.136s sys 0.017s
I20240502 14:14:58.142335 32695 raft_consensus.cc:348] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:58.143107 32695 raft_consensus.cc:729] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:14:58.145320 32695 consensus_queue.cc:260] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 77, Last appended: 1.77, Last appended by leader: 77, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:58.146174 32695 ts_tablet_manager.cc:1398] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
W20240502 14:14:58.202852 30886 kernel_stack_watchdog.cc:198] Thread 30923 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:637 for 339ms:
Kernel stack:
(could not read kernel stack)
User stack:
@ 0x7f2dad6d1980 (unknown) at ??:0
@ 0x90ef77 __sanitizer::internal_sched_yield() at ??:?
@ 0x87462d __interceptor_pthread_create at ??:?
@ 0x7f2daa33fd11 kudu::Thread::StartThread() at ??:0
@ 0xa4d187 kudu::Thread::Create() at ??:?
@ 0x7f2daa3607ab kudu::ThreadPool::CreateThread() at ??:0
@ 0x7f2daa35c3ad kudu::ThreadPool::DoSubmit() at ??:0
@ 0x7f2daa35a84f kudu::ThreadPoolToken::Submit() at ??:0
@ 0x7f2db6ec632a kudu::tablet::OpDriver::ExecuteAsync() at ??:0
@ 0x7f2db6e6cc00 kudu::tablet::TabletReplica::SubmitWrite() at ??:0
@ 0x7f2db9846aa1 kudu::tserver::TabletServiceImpl::Write() at ??:0
@ 0x7f2db3febdc7 kudu::tserver::TabletServerServiceIf::TabletServerServiceIf()::$_3::operator()() at ??:0
@ 0x7f2db3febaf6 std::_Function_handler<>::_M_invoke() at ??:0
@ 0x7f2db102404f std::function<>::operator()() at ??:0
@ 0x7f2db1022f2f kudu::rpc::GeneratedServiceIf::Handle() at ??:0
@ 0x7f2db1029406 kudu::rpc::ServicePool::RunThread() at ??:0
W20240502 14:14:58.204602 30923 thread.cc:636] prepare [worker] (thread pool) Time spent creating pthread: real 0.588s user 0.000s sys 0.009s
W20240502 14:14:58.204703 30923 thread.cc:602] prepare [worker] (thread pool) Time spent starting thread: real 0.588s user 0.000s sys 0.009s
I20240502 14:14:58.232877 32736 tablet_copy_client.cc:523] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:14:58.233628 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-3183d887deec4e40b88dceda3306c0f1 received from {username='slave'} at 127.27.220.135:57521
I20240502 14:14:58.233809 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-3183d887deec4e40b88dceda3306c0f1 on tablet 3183d887deec4e40b88dceda3306c0f1 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:14:58.236112 32736 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:14:58.249847 32578 raft_consensus.cc:1050] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:14:58.250857 32578 consensus_queue.cc:237] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 52, Committed index: 52, Last appended: 1.53, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 54 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:14:58.254032 32158 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 53. Preceding OpId from leader: term: 1 index: 54. (index mismatch)
I20240502 14:14:58.254992 32418 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 54, Last known committed idx: 52, Time since last communication: 0.000s
I20240502 14:14:58.256963 31091 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 52. Preceding OpId from leader: term: 1 index: 54. (index mismatch)
I20240502 14:14:58.257310 30964 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 53. Preceding OpId from leader: term: 1 index: 54. (index mismatch)
I20240502 14:14:58.257726 32578 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 54, Last known committed idx: 52, Time since last communication: 0.000s
I20240502 14:14:58.258000 32361 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 54, Last known committed idx: 52, Time since last communication: 0.000s
I20240502 14:14:58.266999 31092 raft_consensus.cc:1203] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.52->[1.53-1.54] Dedup: 1.53->[1.54-1.54]
I20240502 14:14:58.268806 32418 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 LEADER]: Committing config change with OpId 1.54: config changed from index 41 to 54, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 54 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.276458 32158 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.54: config changed from index 41 to 54, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 54 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:14:58.278093 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 43545.9us
I20240502 14:14:58.278023 30964 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.54: config changed from index 41 to 54, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 54 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.282496 31092 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.54: config changed from index 41 to 54, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 54 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.283147 30431 catalog_manager.cc:5484] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 41 to 54, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "155f26c73dc84441a58417c09f577494" committed_config { opid_index: 54 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:14:58.288055 30701 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 52. Preceding OpId from leader: term: 1 index: 54. (index mismatch)
I20240502 14:14:58.291764 32418 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 54, Last known committed idx: 52, Time since last communication: 0.000s
I20240502 14:14:58.348148 31223 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 53. Preceding OpId from leader: term: 1 index: 54. (index mismatch)
I20240502 14:14:58.356946 32578 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 54, Last known committed idx: 52, Time since last communication: 0.000s
I20240502 14:14:58.384640 30696 raft_consensus.cc:1203] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.52->[1.53-1.54] Dedup: 1.53->[1.54-1.54]
I20240502 14:14:58.387248 31223 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.54: config changed from index 41 to 54, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 54 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.394145 301 raft_consensus.cc:1050] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:14:58.395452 301 consensus_queue.cc:237] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 67, Committed index: 67, Last appended: 1.67, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:14:58.404945 32678 raft_consensus.cc:1013] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: attempt to promote peer 3eafb09239024de3bc0eea8d700f7668: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 14:14:58.412385 30696 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.54: config changed from index 41 to 54, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 54 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:14:58.483395 31027 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 77439.5us
I20240502 14:14:58.484458 30571 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 67. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 14:14:58.484457 32158 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 67. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 14:14:58.485157 32678 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 68, Last known committed idx: 67, Time since last communication: 0.000s
I20240502 14:14:58.485410 32373 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 68, Last known committed idx: 67, Time since last communication: 0.000s
I20240502 14:14:58.492035 30958 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 67. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 14:14:58.492905 30696 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 67. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 14:14:58.493441 32373 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 68, Last known committed idx: 67, Time since last communication: 0.000s
I20240502 14:14:58.495463 32678 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 68, Last known committed idx: 66, Time since last communication: 0.000s
I20240502 14:14:58.498497 30833 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 67. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 14:14:58.499003 32373 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 68, Last known committed idx: 67, Time since last communication: 0.000s
I20240502 14:14:58.504096 32373 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.68: config changed from index 55 to 68, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.508157 30964 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.68: config changed from index 55 to 68, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.513597 30428 catalog_manager.cc:5484] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index 55 to 68, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.513971 30574 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.68: config changed from index 55 to 68, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.518249 32736 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=91 overwritten=0 applied=91 ignored=0} inserts{seen=763 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:14:58.518697 32736 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:14:58.519414 32736 ts_tablet_manager.cc:1367] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.283s user 0.130s sys 0.018s
I20240502 14:14:58.520318 32736 raft_consensus.cc:348] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:58.520702 32736 raft_consensus.cc:729] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:14:58.520910 32736 consensus_queue.cc:260] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 91, Last appended: 1.91, Last appended by leader: 91, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 91 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:58.524451 32158 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.68: config changed from index 55 to 68, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.539157 32736 ts_tablet_manager.cc:1398] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.020s user 0.002s sys 0.000s
I20240502 14:14:58.559434 30832 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.68: config changed from index 55 to 68, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.563927 30830 consensus_queue.cc:237] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 55, Committed index: 55, Last appended: 1.56, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:14:58.567108 31092 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 55. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:58.567253 30961 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 55. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:58.567745 32418 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 54, Time since last communication: 0.000s
I20240502 14:14:58.568085 32578 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 55, Time since last communication: 0.000s
I20240502 14:14:58.590728 31092 consensus_queue.cc:237] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 69, Committed index: 69, Last appended: 1.69, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:14:58.593580 32361 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 LEADER]: Committing config change with OpId 1.57: config changed from index 54 to 57, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.599341 32158 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 14:14:58.605553 301 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 69, Time since last communication: 0.000s
I20240502 14:14:58.605739 30958 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 14:14:58.606540 301 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 69, Time since last communication: 0.000s
I20240502 14:14:58.607841 30417 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0f906c7d775540e6ba76cf2ff3700a67 with cas_config_opid_index 54: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:14:58.609455 30699 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 14:14:58.612519 30833 raft_consensus.cc:1261] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 69. Preceding OpId from leader: term: 1 index: 70. (index mismatch)
I20240502 14:14:58.613649 32765 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 69, Time since last communication: 0.000s
I20240502 14:14:58.614104 32765 consensus_queue.cc:1035] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 70, Last known committed idx: 67, Time since last communication: 0.000s
I20240502 14:14:58.617113 30697 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:58.617238 32158 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 14:14:58.616833 32766 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.70: config changed from index 68 to 70, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.618235 30964 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index 54 to 57, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.622380 30418 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 36b27125eebb410e8c37560450778dd5 with cas_config_opid_index 68: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:14:58.622931 30958 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index 68 to 70, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.623193 32158 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index 68 to 70, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.623437 30828 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index 68 to 70, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.624006 32681 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 55, Time since last communication: 0.000s
I20240502 14:14:58.624583 32681 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 55, Time since last communication: 0.000s
I20240502 14:14:58.628895 30428 catalog_manager.cc:5484] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 68 to 70, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:14:58.611851 31092 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index 54 to 57, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.634315 32157 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index 54 to 57, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.629324 30697 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index 54 to 57, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.638427 30426 catalog_manager.cc:5484] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 54 to 57, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New cstate: current_term: 1 leader_uuid: "155f26c73dc84441a58417c09f577494" committed_config { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:14:58.643574 32156 raft_consensus.cc:1203] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.69->[1.70-1.80] Dedup: 1.77->[1.78-1.80]
I20240502 14:14:58.649139 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-8b1b5e519c314a83bf1e39e011440fa6 received from {username='slave'} at 127.27.220.135:57521
I20240502 14:14:58.649298 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-8b1b5e519c314a83bf1e39e011440fa6 on tablet 8b1b5e519c314a83bf1e39e011440fa6 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:14:58.655247 30553 tablet_service.cc:1514] Processing DeleteTablet for tablet 36b27125eebb410e8c37560450778dd5 with delete_type TABLET_DATA_TOMBSTONED (TS fd3399b1bfe644159a833ede6726c25f not found in new config with opid_index 70) from {username='slave'} at 127.0.0.1:38096
I20240502 14:14:58.627076 30699 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.68: config changed from index 55 to 68, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 68 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.679445 314 tablet_replica.cc:330] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: stopping tablet replica
I20240502 14:14:58.679946 314 raft_consensus.cc:2229] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:14:58.680351 314 raft_consensus.cc:2258] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:14:58.686532 314 ts_tablet_manager.cc:1875] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:14:58.694186 314 ts_tablet_manager.cc:1888] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.69
I20240502 14:14:58.694340 314 log.cc:1196] T 36b27125eebb410e8c37560450778dd5 P fd3399b1bfe644159a833ede6726c25f: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/36b27125eebb410e8c37560450778dd5
I20240502 14:14:58.695389 30419 catalog_manager.cc:4830] TS fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): tablet 36b27125eebb410e8c37560450778dd5 (table rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]) successfully deleted
I20240502 14:14:58.726128 31204 tablet_service.cc:1514] Processing DeleteTablet for tablet 0f906c7d775540e6ba76cf2ff3700a67 with delete_type TABLET_DATA_TOMBSTONED (TS 0dfd48d0b03a42f689e58d5190d03a2f not found in new config with opid_index 57) from {username='slave'} at 127.0.0.1:56766
I20240502 14:14:58.779801 32387 raft_consensus.cc:1050] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:14:58.874667 32158 raft_consensus.cc:1203] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.90->[1.91-1.99] Dedup: 1.91->[1.92-1.99]
I20240502 14:14:59.379438 32662 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: Bootstrap replayed 1/1 log segments. Stats: ops{read=63 overwritten=0 applied=63 ignored=0} inserts{seen=735 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:14:59.380092 32662 tablet_bootstrap.cc:492] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: Bootstrap complete.
I20240502 14:14:59.380574 32662 ts_tablet_manager.cc:1367] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: Time spent bootstrapping tablet: real 1.613s user 0.116s sys 0.020s
I20240502 14:14:59.381397 32662 raft_consensus.cc:348] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } }
I20240502 14:14:59.439218 32662 raft_consensus.cc:729] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 155f26c73dc84441a58417c09f577494, State: Initialized, Role: LEARNER
I20240502 14:14:59.439543 32662 consensus_queue.cc:260] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 63, Last appended: 1.63, Last appended by leader: 63, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: true } }
I20240502 14:14:59.440284 32662 ts_tablet_manager.cc:1398] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494: Time spent starting tablet: real 0.060s user 0.002s sys 0.000s
I20240502 14:14:58.816282 30699 raft_consensus.cc:2932] T 36b27125eebb410e8c37560450778dd5 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.70: config changed from index 68 to 70, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 70 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:58.781294 32387 consensus_queue.cc:237] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 82, Committed index: 82, Last appended: 1.82, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:14:59.446714 32158 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 82. Preceding OpId from leader: term: 1 index: 83. (index mismatch)
I20240502 14:14:59.446849 30964 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 82. Preceding OpId from leader: term: 1 index: 83. (index mismatch)
I20240502 14:14:59.174701 32677 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: Bootstrap replayed 1/1 log segments. Stats: ops{read=61 overwritten=0 applied=61 ignored=0} inserts{seen=733 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:14:59.447556 32677 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: Bootstrap complete.
I20240502 14:14:59.448143 32677 ts_tablet_manager.cc:1367] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: Time spent bootstrapping tablet: real 1.808s user 0.129s sys 0.093s
I20240502 14:14:58.783404 319 tablet_replica.cc:330] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: stopping tablet replica
I20240502 14:14:58.842904 31223 consensus_queue.cc:237] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 92, Committed index: 92, Last appended: 1.92, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:14:59.328040 32661 raft_consensus.cc:1050] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:14:59.450778 32661 consensus_queue.cc:237] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 99, Committed index: 99, Last appended: 1.99, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 100 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:14:59.453706 30573 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 92. Preceding OpId from leader: term: 1 index: 93. (index mismatch)
I20240502 14:14:59.454326 32454 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 93, Last known committed idx: 92, Time since last communication: 0.000s
I20240502 14:14:59.454936 319 raft_consensus.cc:2229] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:14:59.455349 319 pending_rounds.cc:62] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: Trying to abort 1 pending ops.
I20240502 14:14:59.455560 30829 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 92. Preceding OpId from leader: term: 1 index: 93. (index mismatch)
I20240502 14:14:59.455466 319 pending_rounds.cc:68] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: Aborting op as it isn't in flight: id { term: 1 index: 55 } timestamp: 7023244486772129792 op_type: WRITE_OP write_request { tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\270\215\317S\207QVo\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""c21d3bee9a935056" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023244486645260288 authz_token { token_data: "\010\203\302\316\261\006\"3\n\005slave\022*\n 68eff6e9fd9a4bf0a49dc0d8c6a7ef37\020\001\030\001 \001(\001" signature: "<redacted>""\024\305\205\347\363\300\236;\325\352G\301U\261z\272\311@\322\224T\331\367\037K{\347\034\025\335\215\303\035}f\203\210\230\363wv;uZ\303\336xf\217n5+\275\277\320\234t\326y\347\331X\304U" signing_key_seq_num: 0 } } request_id { client_id: "def8668d6d4e4e0da9193a8148b8890c" seq_no: 161 first_incomplete_seq_no: 161 attempt_no: 0 }
I20240502 14:14:59.456115 32710 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 83, Last known committed idx: 82, Time since last communication: 0.000s
I20240502 14:14:59.456350 319 raft_consensus.cc:2258] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:14:59.456458 32454 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 93, Last known committed idx: 92, Time since last communication: 0.000s
I20240502 14:14:59.456779 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 155f26c73dc84441a58417c09f577494-d4aa1eb08d4b450f9c5be499d8c6bbc7 received from {username='slave'} at 127.27.220.131:33683
I20240502 14:14:59.456935 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 155f26c73dc84441a58417c09f577494-d4aa1eb08d4b450f9c5be499d8c6bbc7 on tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 with peer 155f26c73dc84441a58417c09f577494
I20240502 14:14:59.457226 30833 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 82. Preceding OpId from leader: term: 1 index: 83. (index mismatch)
I20240502 14:14:59.458236 32589 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 83, Last known committed idx: 82, Time since last communication: 0.000s
I20240502 14:14:59.459249 30696 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 82. Preceding OpId from leader: term: 1 index: 83. (index mismatch)
I20240502 14:14:59.461400 32677 raft_consensus.cc:348] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } }
I20240502 14:14:59.461844 32677 raft_consensus.cc:729] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d733c51902044956a8cd2b6b31bc9831, State: Initialized, Role: LEARNER
I20240502 14:14:59.462142 32677 consensus_queue.cc:260] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 61, Last appended: 1.61, Last appended by leader: 61, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } }
I20240502 14:14:59.465083 30571 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 99. Preceding OpId from leader: term: 1 index: 100. (index mismatch)
I20240502 14:14:59.465440 32454 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 83, Last known committed idx: 82, Time since last communication: 0.000s
I20240502 14:14:59.466215 30704 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 99. Preceding OpId from leader: term: 1 index: 100. (index mismatch)
I20240502 14:14:59.466851 32156 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 99. Preceding OpId from leader: term: 1 index: 100. (index mismatch)
I20240502 14:14:59.467228 32454 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 100, Last known committed idx: 99, Time since last communication: 0.000s
I20240502 14:14:59.473356 319 ts_tablet_manager.cc:1875] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:14:59.478456 30832 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 99. Preceding OpId from leader: term: 1 index: 100. (index mismatch)
I20240502 14:14:59.479025 31092 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 82. Preceding OpId from leader: term: 1 index: 83. (index mismatch)
I20240502 14:14:59.480123 32279 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 83, Last known committed idx: 82, Time since last communication: 0.000s
I20240502 14:14:59.480715 32279 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 83, Last known committed idx: 82, Time since last communication: 0.000s
I20240502 14:14:59.482018 30958 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 92. Preceding OpId from leader: term: 1 index: 93. (index mismatch)
W20240502 14:14:59.482488 31157 consensus_peers.cc:485] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 356c1606c56147a2868e60ab63c59c13. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:14:59.483173 31089 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 99. Preceding OpId from leader: term: 1 index: 100. (index mismatch)
I20240502 14:14:59.483341 32710 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 100, Last known committed idx: 99, Time since last communication: 0.000s
I20240502 14:14:59.483907 32387 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 100, Last known committed idx: 99, Time since last communication: 0.000s
I20240502 14:14:59.484423 32387 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 100, Last known committed idx: 99, Time since last communication: 0.000s
I20240502 14:14:59.502365 30700 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 92. Preceding OpId from leader: term: 1 index: 93. (index mismatch)
I20240502 14:14:59.505506 30696 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.100: config changed from index 91 to 100, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 100 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.510577 30701 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.83: config changed from index 70 to 83, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.525856 32387 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.93: config changed from index -1 to 93, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:59.529173 32707 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 93, Last known committed idx: 91, Time since last communication: 0.000s
I20240502 14:14:59.529872 32707 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 100, Last known committed idx: 99, Time since last communication: 0.000s
I20240502 14:14:59.530009 32279 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 93, Last known committed idx: 92, Time since last communication: 0.000s
I20240502 14:14:59.533327 32156 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.100: config changed from index 91 to 100, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 100 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.537459 30829 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.93: config changed from index -1 to 93, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:59.544025 30571 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.93: config changed from index -1 to 93, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:59.540352 30704 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.93: config changed from index -1 to 93, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:59.549063 31092 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.100: config changed from index 91 to 100, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 100 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.559880 30964 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.93: config changed from index -1 to 93, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:59.564958 32677 ts_tablet_manager.cc:1398] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831: Time spent starting tablet: real 0.117s user 0.012s sys 0.000s
I20240502 14:14:59.564947 30426 catalog_manager.cc:5484] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 reported cstate change: config changed from index 91 to 100, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 100 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.565856 31114 tablet_copy_service.cc:342] P ba420f2442d6416bbda95edfff85e849: Request end of tablet copy session d733c51902044956a8cd2b6b31bc9831-3c26d346223a498bb96b962fa7dd99aa received from {username='slave'} at 127.27.220.130:36045
I20240502 14:14:59.565979 31114 tablet_copy_service.cc:433] P ba420f2442d6416bbda95edfff85e849: ending tablet copy session d733c51902044956a8cd2b6b31bc9831-3c26d346223a498bb96b962fa7dd99aa on tablet 3c26d346223a498bb96b962fa7dd99aa with peer d733c51902044956a8cd2b6b31bc9831
I20240502 14:14:59.504685 32715 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.83: config changed from index 70 to 83, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.618182 30696 raft_consensus.cc:1203] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.56->[1.57-1.74] Dedup: 1.61->[1.62-1.74]
W20240502 14:14:59.631847 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 88093.3us
I20240502 14:14:59.634986 30958 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.83: config changed from index 70 to 83, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.638722 31092 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.83: config changed from index 70 to 83, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.654268 30428 catalog_manager.cc:5484] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 reported cstate change: config changed from index 70 to 83, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.654949 319 ts_tablet_manager.cc:1888] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.55
I20240502 14:14:59.655164 319 log.cc:1196] T 0f906c7d775540e6ba76cf2ff3700a67 P 0dfd48d0b03a42f689e58d5190d03a2f: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/wals/0f906c7d775540e6ba76cf2ff3700a67
I20240502 14:14:59.654949 30428 catalog_manager.cc:5484] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 reported cstate change: config changed from index -1 to 93, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:14:59.504506 32710 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.100: config changed from index 91 to 100, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 100 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:14:59.685963 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 128840us
I20240502 14:14:59.689656 30420 catalog_manager.cc:4830] TS 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403): tablet 0f906c7d775540e6ba76cf2ff3700a67 (table rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]) successfully deleted
I20240502 14:14:59.690269 32156 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.83: config changed from index 70 to 83, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.695688 30828 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.83: config changed from index 70 to 83, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 83 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.718940 30833 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.100: config changed from index 91 to 100, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 100 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:14:59.733043 32751 log.cc:924] Time spent T 00000000000000000000000000000000 P 1cb98ed728ea452ab7c2d0de3cb89c33: Append to log took a long time: real 0.076s user 0.000s sys 0.000s
W20240502 14:14:59.736160 31159 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.129:33055, user_credentials={real_user=slave}} blocked reactor thread for 184433us
I20240502 14:14:59.737695 30573 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.100: config changed from index 91 to 100, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 100 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.799830 31223 consensus_queue.cc:237] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 84, Committed index: 84, Last appended: 1.84, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:14:59.808344 32156 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:14:59.809891 30961 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:14:59.810192 30827 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:14:59.810403 32589 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:14:59.812098 31222 consensus_queue.cc:237] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 100, Committed index: 100, Last appended: 1.100, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:14:59.813251 368 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:14:59.813714 368 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:14:59.814889 30704 raft_consensus.cc:1261] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:14:59.815668 32589 consensus_queue.cc:1035] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:14:59.818915 30701 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 100. Preceding OpId from leader: term: 1 index: 101. (index mismatch)
I20240502 14:14:59.819226 31089 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 100. Preceding OpId from leader: term: 1 index: 101. (index mismatch)
I20240502 14:14:59.820688 32156 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 100. Preceding OpId from leader: term: 1 index: 101. (index mismatch)
I20240502 14:14:59.821635 32589 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 101, Last known committed idx: 100, Time since last communication: 0.000s
I20240502 14:14:59.827247 30572 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 100. Preceding OpId from leader: term: 1 index: 101. (index mismatch)
I20240502 14:14:59.830449 32715 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 101, Last known committed idx: 100, Time since last communication: 0.000s
I20240502 14:14:59.831348 372 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 101, Last known committed idx: 100, Time since last communication: 0.000s
I20240502 14:14:59.830907 32661 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.85: config changed from index 83 to 85, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.836433 32279 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.101: config changed from index 100 to 101, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.841437 32279 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 101, Last known committed idx: 100, Time since last communication: 0.000s
I20240502 14:14:59.842016 30829 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 83 to 85, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.844386 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3183d887deec4e40b88dceda3306c0f1 with cas_config_opid_index 83: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:14:59.844462 32156 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 83 to 85, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.840867 30573 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.101: config changed from index 100 to 101, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.846726 31089 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.101: config changed from index 100 to 101, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.849088 32157 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.101: config changed from index 100 to 101, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.854765 30433 catalog_manager.cc:5484] T 3183d887deec4e40b88dceda3306c0f1 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 83 to 85, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.859758 30427 catalog_manager.cc:5484] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 100 to 101, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.861980 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8b1b5e519c314a83bf1e39e011440fa6 with cas_config_opid_index 100: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:14:59.864701 31072 tablet_service.cc:1514] Processing DeleteTablet for tablet 3183d887deec4e40b88dceda3306c0f1 with delete_type TABLET_DATA_TOMBSTONED (TS ba420f2442d6416bbda95edfff85e849 not found in new config with opid_index 85) from {username='slave'} at 127.0.0.1:42598
I20240502 14:14:59.870512 30704 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.101: config changed from index 100 to 101, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 101 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.877192 30833 raft_consensus.cc:1203] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.56->[1.57-1.82] Dedup: 1.63->[1.64-1.82]
I20240502 14:14:59.880147 30699 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 83 to 85, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.889094 301 raft_consensus.cc:1050] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: attempting to promote NON_VOTER d733c51902044956a8cd2b6b31bc9831 to VOTER
I20240502 14:14:59.889186 381 raft_consensus.cc:1050] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: attempting to promote NON_VOTER d733c51902044956a8cd2b6b31bc9831 to VOTER
I20240502 14:14:59.890295 301 consensus_queue.cc:237] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 75, Committed index: 75, Last appended: 1.75, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } }
W20240502 14:14:59.892611 381 raft_consensus.cc:1054] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: Unable to promote non-voter d733c51902044956a8cd2b6b31bc9831: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: true } }.
Pending config: opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } }
I20240502 14:14:59.895154 31223 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 75. Preceding OpId from leader: term: 1 index: 76. (index mismatch)
I20240502 14:14:59.895865 30573 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 75. Preceding OpId from leader: term: 1 index: 76. (index mismatch)
I20240502 14:14:59.896338 30964 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 75. Preceding OpId from leader: term: 1 index: 76. (index mismatch)
I20240502 14:14:59.896548 32373 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 76, Last known committed idx: 75, Time since last communication: 0.000s
I20240502 14:14:59.897058 32373 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 76, Last known committed idx: 75, Time since last communication: 0.000s
I20240502 14:14:59.897161 30704 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 LEARNER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 75. Preceding OpId from leader: term: 1 index: 76. (index mismatch)
I20240502 14:14:59.906224 301 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 76, Last known committed idx: 75, Time since last communication: 0.000s
I20240502 14:14:59.910768 32373 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.76: config changed from index 57 to 76, d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:14:59.915489 31223 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.76: config changed from index 57 to 76, d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:14:59.916524 30573 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.76: config changed from index 57 to 76, d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:14:59.921591 32373 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 76, Last known committed idx: 75, Time since last communication: 0.000s
I20240502 14:14:59.926191 30961 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.76: config changed from index 57 to 76, d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:14:59.926354 30427 catalog_manager.cc:5484] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 57 to 76, d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:14:59.932231 30704 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.76: config changed from index 57 to 76, d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:14:59.885190 30813 tablet_service.cc:1514] Processing DeleteTablet for tablet 8b1b5e519c314a83bf1e39e011440fa6 with delete_type TABLET_DATA_TOMBSTONED (TS 155f26c73dc84441a58417c09f577494 not found in new config with opid_index 101) from {username='slave'} at 127.0.0.1:33764
I20240502 14:14:59.904371 30828 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 75. Preceding OpId from leader: term: 1 index: 76. (index mismatch)
I20240502 14:14:59.948624 32373 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 76, Last known committed idx: 75, Time since last communication: 0.000s
I20240502 14:14:59.951759 30828 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.76: config changed from index 57 to 76, d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 76 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:14:59.957893 30961 raft_consensus.cc:2932] T 3183d887deec4e40b88dceda3306c0f1 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 83 to 85, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:14:59.980270 387 tablet_replica.cc:330] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: stopping tablet replica
I20240502 14:14:59.980798 387 raft_consensus.cc:2229] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:00.055521 387 raft_consensus.cc:2258] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:00.068953 31094 consensus_queue.cc:237] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 77, Committed index: 77, Last appended: 1.77, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } }
I20240502 14:15:00.074386 388 ts_tablet_manager.cc:902] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:15:00.078577 31222 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 77. Preceding OpId from leader: term: 1 index: 78. (index mismatch)
I20240502 14:15:00.078598 30701 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 77. Preceding OpId from leader: term: 1 index: 78. (index mismatch)
I20240502 14:15:00.079931 387 ts_tablet_manager.cc:1875] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:00.081096 30573 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 77. Preceding OpId from leader: term: 1 index: 78. (index mismatch)
I20240502 14:15:00.082912 30827 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 77. Preceding OpId from leader: term: 1 index: 78. (index mismatch)
I20240502 14:15:00.088394 389 tablet_replica.cc:330] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: stopping tablet replica
I20240502 14:15:00.088904 389 raft_consensus.cc:2229] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:00.089924 389 raft_consensus.cc:2258] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:00.091514 389 ts_tablet_manager.cc:1875] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:00.082679 32373 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 78, Last known committed idx: 77, Time since last communication: 0.000s
I20240502 14:15:00.093886 301 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 78, Last known committed idx: 77, Time since last communication: 0.000s
I20240502 14:15:00.094175 32766 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 78, Last known committed idx: 77, Time since last communication: 0.000s
I20240502 14:15:00.094594 381 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 78, Last known committed idx: 77, Time since last communication: 0.000s
I20240502 14:15:00.100116 32766 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.78: config changed from index 76 to 78, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:15:00.101471 30827 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.78: config changed from index 76 to 78, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:15:00.104641 30418 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3c26d346223a498bb96b962fa7dd99aa with cas_config_opid_index 76: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:00.106021 31222 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.78: config changed from index 76 to 78, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:15:00.107987 30433 catalog_manager.cc:5484] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 76 to 78, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:15:00.113453 30704 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.78: config changed from index 76 to 78, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:15:00.133489 30569 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.78: config changed from index 76 to 78, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 78 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } }
I20240502 14:15:00.148499 387 ts_tablet_manager.cc:1888] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.84
I20240502 14:15:00.160843 30943 tablet_service.cc:1514] Processing DeleteTablet for tablet 3c26d346223a498bb96b962fa7dd99aa with delete_type TABLET_DATA_TOMBSTONED (TS 7c1f568f8efc4b2a9dbf38907f02f3e3 not found in new config with opid_index 78) from {username='slave'} at 127.0.0.1:53120
I20240502 14:15:00.177727 387 log.cc:1196] T 3183d887deec4e40b88dceda3306c0f1 P ba420f2442d6416bbda95edfff85e849: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/wals/3183d887deec4e40b88dceda3306c0f1
I20240502 14:15:00.188129 30418 catalog_manager.cc:4830] TS ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805): tablet 3183d887deec4e40b88dceda3306c0f1 (table rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]) successfully deleted
I20240502 14:15:00.249048 400 tablet_replica.cc:330] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: stopping tablet replica
I20240502 14:15:00.292022 400 raft_consensus.cc:2229] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:00.292640 400 raft_consensus.cc:2258] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:00.317759 388 tablet_copy_client.cc:308] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:15:00.319370 400 ts_tablet_manager.cc:1875] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
W20240502 14:15:00.323076 30510 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.134:39403, user_credentials={real_user=slave}} blocked reactor thread for 107262us
I20240502 14:15:00.339426 389 ts_tablet_manager.cc:1888] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.100
I20240502 14:15:00.341442 31243 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet 356c1606c56147a2868e60ab63c59c13 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:57521)
I20240502 14:15:00.341689 31243 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet 356c1606c56147a2868e60ab63c59c13 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:57521: session id = 3eafb09239024de3bc0eea8d700f7668-356c1606c56147a2868e60ab63c59c13
I20240502 14:15:00.344694 31243 tablet_copy_source_session.cc:205] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:00.344944 389 log.cc:1196] T 8b1b5e519c314a83bf1e39e011440fa6 P 155f26c73dc84441a58417c09f577494: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/8b1b5e519c314a83bf1e39e011440fa6
I20240502 14:15:00.346702 400 ts_tablet_manager.cc:1888] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.77
I20240502 14:15:00.346920 400 log.cc:1196] T 3c26d346223a498bb96b962fa7dd99aa P 7c1f568f8efc4b2a9dbf38907f02f3e3: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/3c26d346223a498bb96b962fa7dd99aa
I20240502 14:15:00.348856 30418 catalog_manager.cc:4830] TS 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): tablet 3c26d346223a498bb96b962fa7dd99aa (table rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]) successfully deleted
I20240502 14:15:00.351972 388 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 356c1606c56147a2868e60ab63c59c13. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:00.353842 30417 catalog_manager.cc:4830] TS 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): tablet 8b1b5e519c314a83bf1e39e011440fa6 (table rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]) successfully deleted
I20240502 14:15:00.359582 388 tablet_copy_client.cc:783] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:00.359841 388 tablet_copy_client.cc:647] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:00.422731 32454 raft_consensus.cc:1050] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 155f26c73dc84441a58417c09f577494 to VOTER
I20240502 14:15:00.423714 32454 consensus_queue.cc:237] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 84, Committed index: 84, Last appended: 1.84, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } }
W20240502 14:15:00.424350 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 64275.9us
I20240502 14:15:00.431672 31091 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:15:00.438421 368 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:15:00.441176 30570 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:15:00.445629 32279 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:15:00.465715 30961 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:15:00.492972 30827 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494 [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:15:00.497337 30696 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 84. Preceding OpId from leader: term: 1 index: 85. (index mismatch)
I20240502 14:15:00.504838 388 tablet_copy_client.cc:523] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:00.510797 388 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:15:00.533861 32454 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:15:00.553273 361 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.85: config changed from index 57 to 85, 155f26c73dc84441a58417c09f577494 (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.559155 31094 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 57 to 85, 155f26c73dc84441a58417c09f577494 (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.565006 32589 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:15:00.565887 30433 catalog_manager.cc:5484] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 57 to 85, 155f26c73dc84441a58417c09f577494 (127.27.220.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.567620 32589 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 85, Last known committed idx: 84, Time since last communication: 0.000s
I20240502 14:15:00.576570 30573 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 57 to 85, 155f26c73dc84441a58417c09f577494 (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.584614 30833 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 57 to 85, 155f26c73dc84441a58417c09f577494 (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.593986 30696 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 57 to 85, 155f26c73dc84441a58417c09f577494 (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
W20240502 14:15:00.629194 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 53325.8us
I20240502 14:15:00.637997 30958 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.85: config changed from index 57 to 85, 155f26c73dc84441a58417c09f577494 (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 85 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.723251 31223 consensus_queue.cc:237] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 86, Committed index: 86, Last appended: 1.86, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 87 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } }
I20240502 14:15:00.729359 30832 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 86. Preceding OpId from leader: term: 1 index: 87. (index mismatch)
I20240502 14:15:00.732473 30572 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 86. Preceding OpId from leader: term: 1 index: 87. (index mismatch)
I20240502 14:15:00.734238 32454 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 87, Last known committed idx: 86, Time since last communication: 0.000s
I20240502 14:15:00.734777 30961 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 86. Preceding OpId from leader: term: 1 index: 87. (index mismatch)
I20240502 14:15:00.738173 30696 raft_consensus.cc:1261] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 86. Preceding OpId from leader: term: 1 index: 87. (index mismatch)
I20240502 14:15:00.741683 32454 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 87, Last known committed idx: 86, Time since last communication: 0.000s
I20240502 14:15:00.748471 361 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 87, Last known committed idx: 85, Time since last communication: 0.000s
I20240502 14:15:00.767705 32589 consensus_queue.cc:1035] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 87, Last known committed idx: 86, Time since last communication: 0.000s
I20240502 14:15:00.799649 32715 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.87: config changed from index 85 to 87, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 87 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.821290 30964 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.87: config changed from index 85 to 87, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 87 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.823956 388 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=96 overwritten=0 applied=96 ignored=0} inserts{seen=727 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:00.824366 388 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:15:00.834210 388 ts_tablet_manager.cc:1367] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.324s user 0.130s sys 0.029s
I20240502 14:15:00.835438 388 raft_consensus.cc:348] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:00.835721 388 raft_consensus.cc:729] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:15:00.835988 388 consensus_queue.cc:260] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 96, Last appended: 1.96, Last appended by leader: 96, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 93 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:00.836570 30574 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.87: config changed from index 85 to 87, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 87 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.839272 30433 catalog_manager.cc:5484] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index 85 to 87, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 87 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.840898 388 ts_tablet_manager.cc:1398] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.007s user 0.001s sys 0.004s
I20240502 14:15:00.847280 30699 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.87: config changed from index 85 to 87, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 87 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.852440 31072 tablet_service.cc:1514] Processing DeleteTablet for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 with delete_type TABLET_DATA_TOMBSTONED (TS ba420f2442d6416bbda95edfff85e849 not found in new config with opid_index 87) from {username='slave'} at 127.0.0.1:42598
I20240502 14:15:00.895321 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 with cas_config_opid_index 85: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:00.899273 30827 raft_consensus.cc:2932] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.87: config changed from index 85 to 87, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 87 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { promote: false } } }
I20240502 14:15:00.918743 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-356c1606c56147a2868e60ab63c59c13 received from {username='slave'} at 127.27.220.135:57521
I20240502 14:15:00.918900 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-356c1606c56147a2868e60ab63c59c13 on tablet 356c1606c56147a2868e60ab63c59c13 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:15:01.026645 424 tablet_replica.cc:330] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: stopping tablet replica
I20240502 14:15:01.026885 424 raft_consensus.cc:2229] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:01.033703 424 raft_consensus.cc:2258] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Raft consensus is shut down!
W20240502 14:15:01.057051 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 46691.2us
I20240502 14:15:01.068940 424 ts_tablet_manager.cc:1875] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:01.089798 424 ts_tablet_manager.cc:1888] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.86
I20240502 14:15:01.090045 424 log.cc:1196] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P ba420f2442d6416bbda95edfff85e849: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/wals/d4aa1eb08d4b450f9c5be499d8c6bbc7
I20240502 14:15:01.092406 30418 catalog_manager.cc:4830] TS ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805): tablet d4aa1eb08d4b450f9c5be499d8c6bbc7 (table rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]) successfully deleted
I20240502 14:15:01.245167 32156 raft_consensus.cc:1203] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.92->[1.93-1.103] Dedup: 1.96->[1.97-1.103]
I20240502 14:15:01.415704 32715 raft_consensus.cc:1050] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:15:01.416657 32715 consensus_queue.cc:237] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 104, Committed index: 104, Last appended: 1.104, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:01.424301 30960 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
I20240502 14:15:01.425940 368 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 104, Time since last communication: 0.000s
I20240502 14:15:01.431808 30570 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
I20240502 14:15:01.435909 32715 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 104, Time since last communication: 0.002s
I20240502 14:15:01.485599 30829 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
I20240502 14:15:01.486871 30700 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
I20240502 14:15:01.488164 32156 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
I20240502 14:15:01.509826 32589 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 103, Time since last communication: 0.000s
I20240502 14:15:01.537375 372 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 104, Time since last communication: 0.000s
I20240502 14:15:01.541738 32715 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 103, Time since last communication: 0.000s
I20240502 14:15:01.608042 372 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.105: config changed from index 93 to 105, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:01.637253 30573 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index 93 to 105, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:01.639118 30964 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index 93 to 105, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:01.650051 32158 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index 93 to 105, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:01.650358 30433 catalog_manager.cc:5484] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 93 to 105, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:01.659915 30700 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index 93 to 105, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:01.681536 30827 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index 93 to 105, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:01.926263 31222 consensus_queue.cc:237] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 108, Committed index: 108, Last appended: 1.108, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 109 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:01.955929 32156 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 108. Preceding OpId from leader: term: 1 index: 109. (index mismatch)
I20240502 14:15:01.962363 30829 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 108. Preceding OpId from leader: term: 1 index: 109. (index mismatch)
I20240502 14:15:01.966732 32454 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 109, Last known committed idx: 108, Time since last communication: 0.000s
I20240502 14:15:01.970844 30704 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 108. Preceding OpId from leader: term: 1 index: 109. (index mismatch)
I20240502 14:15:01.995383 32454 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 109, Last known committed idx: 108, Time since last communication: 0.000s
I20240502 14:15:02.003726 32715 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 109, Last known committed idx: 108, Time since last communication: 0.000s
I20240502 14:15:02.005411 30964 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 108. Preceding OpId from leader: term: 1 index: 109. (index mismatch)
I20240502 14:15:02.020059 368 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 109, Last known committed idx: 108, Time since last communication: 0.000s
I20240502 14:15:02.048027 368 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.109: config changed from index 105 to 109, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 109 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:02.058852 30964 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.109: config changed from index 105 to 109, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 109 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:02.074769 30428 catalog_manager.cc:5484] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index 105 to 109, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 109 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:02.075963 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 356c1606c56147a2868e60ab63c59c13 with cas_config_opid_index 105: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:02.085158 30553 tablet_service.cc:1514] Processing DeleteTablet for tablet 356c1606c56147a2868e60ab63c59c13 with delete_type TABLET_DATA_TOMBSTONED (TS fd3399b1bfe644159a833ede6726c25f not found in new config with opid_index 109) from {username='slave'} at 127.0.0.1:38096
I20240502 14:15:02.086005 473 tablet_replica.cc:330] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: stopping tablet replica
I20240502 14:15:02.103076 473 raft_consensus.cc:2229] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:02.105592 473 raft_consensus.cc:2258] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:02.106437 30704 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.109: config changed from index 105 to 109, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 109 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:02.123003 473 ts_tablet_manager.cc:1875] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:02.131127 473 ts_tablet_manager.cc:1888] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.108
I20240502 14:15:02.131254 473 log.cc:1196] T 356c1606c56147a2868e60ab63c59c13 P fd3399b1bfe644159a833ede6726c25f: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/356c1606c56147a2868e60ab63c59c13
I20240502 14:15:02.132107 30419 catalog_manager.cc:4830] TS fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): tablet 356c1606c56147a2868e60ab63c59c13 (table rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]) successfully deleted
I20240502 14:15:02.240972 30829 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.109: config changed from index 105 to 109, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 109 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:02.298794 32156 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.109: config changed from index 105 to 109, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 109 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:02.650441 31222 consensus_queue.cc:237] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 74, Committed index: 74, Last appended: 1.74, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:02.653949 30572 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 74. Preceding OpId from leader: term: 1 index: 75. (index mismatch)
I20240502 14:15:02.654500 32454 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 75, Last known committed idx: 74, Time since last communication: 0.000s
I20240502 14:15:02.662791 30961 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 74. Preceding OpId from leader: term: 1 index: 75. (index mismatch)
I20240502 14:15:02.663342 32715 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 75, Last known committed idx: 74, Time since last communication: 0.000s
I20240502 14:15:02.664431 30702 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 74. Preceding OpId from leader: term: 1 index: 75. (index mismatch)
I20240502 14:15:02.673624 32454 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.75: config changed from index -1 to 75, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.679688 31089 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 74. Preceding OpId from leader: term: 1 index: 75. (index mismatch)
I20240502 14:15:02.684728 30961 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.75: config changed from index -1 to 75, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.693480 30574 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.75: config changed from index -1 to 75, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.696007 30431 catalog_manager.cc:5484] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index -1 to 75, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.713713 361 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 75, Last known committed idx: 74, Time since last communication: 0.032s
I20240502 14:15:02.715708 361 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 75, Last known committed idx: 74, Time since last communication: 0.000s
I20240502 14:15:02.720181 31089 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.75: config changed from index -1 to 75, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
W20240502 14:15:02.728752 31157 consensus_peers.cc:485] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 23cf772ace1c44ecbd450e0bce820c3f. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:02.863560 30702 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.75: config changed from index -1 to 75, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.930536 30960 consensus_queue.cc:237] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 163, Committed index: 163, Last appended: 1.163, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:02.938493 31091 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 163. Preceding OpId from leader: term: 1 index: 164. (index mismatch)
I20240502 14:15:02.939293 415 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 164, Last known committed idx: 163, Time since last communication: 0.000s
I20240502 14:15:02.944052 30827 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 163. Preceding OpId from leader: term: 1 index: 164. (index mismatch)
I20240502 14:15:02.945693 30574 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 163. Preceding OpId from leader: term: 1 index: 164. (index mismatch)
I20240502 14:15:02.947937 415 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 164, Last known committed idx: 163, Time since last communication: 0.000s
I20240502 14:15:02.950492 346 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.164: config changed from index -1 to 164, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.954443 30573 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.164: config changed from index -1 to 164, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.962527 31089 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.164: config changed from index -1 to 164, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.969359 30424 catalog_manager.cc:5484] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index -1 to 164, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:02.995975 30702 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 163. Preceding OpId from leader: term: 1 index: 164. (index mismatch)
I20240502 14:15:02.996845 413 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 164, Last known committed idx: 162, Time since last communication: 0.000s
I20240502 14:15:03.019902 415 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 164, Last known committed idx: 162, Time since last communication: 0.000s
I20240502 14:15:03.021255 30696 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.164: config changed from index -1 to 164, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.023222 30829 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.164: config changed from index -1 to 164, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
W20240502 14:15:03.055166 30897 consensus_peers.cc:485] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 74c46ec356414d26987e1cd6a0857602. This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:03.059041 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 49018.9us
W20240502 14:15:03.083165 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:41511, user_credentials={real_user=slave}} blocked reactor thread for 56161.1us
W20240502 14:15:03.307730 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 55632.2us
I20240502 14:15:03.368098 521 ts_tablet_manager.cc:902] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:15:03.370970 30964 consensus_queue.cc:237] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 190, Committed index: 190, Last appended: 1.190, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:03.374019 30569 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 190. Preceding OpId from leader: term: 1 index: 191. (index mismatch)
I20240502 14:15:03.374190 31223 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 190. Preceding OpId from leader: term: 1 index: 191. (index mismatch)
I20240502 14:15:03.374712 415 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 191, Last known committed idx: 190, Time since last communication: 0.000s
I20240502 14:15:03.380067 413 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 191, Last known committed idx: 190, Time since last communication: 0.000s
I20240502 14:15:03.388327 32426 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.191: config changed from index -1 to 191, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.388839 31223 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.191: config changed from index -1 to 191, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.394852 30424 catalog_manager.cc:5484] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: config changed from index -1 to 191, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.394780 30569 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.191: config changed from index -1 to 191, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.466558 30827 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 190. Preceding OpId from leader: term: 1 index: 191. (index mismatch)
I20240502 14:15:03.472546 30701 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 190. Preceding OpId from leader: term: 1 index: 191. (index mismatch)
I20240502 14:15:03.477443 519 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 191, Last known committed idx: 190, Time since last communication: 0.000s
I20240502 14:15:03.478150 520 ts_tablet_manager.cc:902] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:15:03.478435 30964 consensus_queue.cc:237] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 105, Committed index: 105, Last appended: 1.105, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:03.485810 31222 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 105. Preceding OpId from leader: term: 1 index: 106. (index mismatch)
I20240502 14:15:03.486645 519 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 106, Last known committed idx: 105, Time since last communication: 0.000s
I20240502 14:15:03.488349 30574 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 106. (index mismatch)
I20240502 14:15:03.488992 519 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 106, Last known committed idx: 104, Time since last communication: 0.000s
W20240502 14:15:03.491067 30897 consensus_peers.cc:485] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5c4ceedcb00a49e1a32977e4e1acea24. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:03.492048 519 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 191, Last known committed idx: 190, Time since last communication: 0.000s
I20240502 14:15:03.493716 31092 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 105. Preceding OpId from leader: term: 1 index: 106. (index mismatch)
I20240502 14:15:03.494182 30828 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 106. (index mismatch)
I20240502 14:15:03.494323 519 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 106, Last known committed idx: 105, Time since last communication: 0.000s
I20240502 14:15:03.499183 30696 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.191: config changed from index -1 to 191, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.502928 346 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.106: config changed from index -1 to 106, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.504110 31092 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.106: config changed from index -1 to 106, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.506907 30829 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.191: config changed from index -1 to 191, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.509115 31222 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.106: config changed from index -1 to 106, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.510113 30573 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.106: config changed from index -1 to 106, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.516438 30424 catalog_manager.cc:5484] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: config changed from index -1 to 106, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
W20240502 14:15:03.551864 30897 consensus_peers.cc:485] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 50f7b19e515e497c9a27e8e94cb73079. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:03.570037 346 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 106, Last known committed idx: 104, Time since last communication: 0.000s
I20240502 14:15:03.615231 30833 raft_consensus.cc:1203] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.104->[1.105-1.106] Dedup: 1.105->[1.106-1.106]
I20240502 14:15:03.615756 30833 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.106: config changed from index -1 to 106, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:03.684262 30569 raft_consensus.cc:1203] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.104->[1.105-1.105] Dedup: 1.105->[]
W20240502 14:15:03.740089 31027 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 53370.1us
I20240502 14:15:03.770989 521 tablet_copy_client.cc:308] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:15:03.773334 31243 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet 23cf772ace1c44ecbd450e0bce820c3f from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:57521)
I20240502 14:15:03.773502 31243 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet 23cf772ace1c44ecbd450e0bce820c3f from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:57521: session id = 3eafb09239024de3bc0eea8d700f7668-23cf772ace1c44ecbd450e0bce820c3f
I20240502 14:15:03.776394 31243 tablet_copy_source_session.cc:205] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:03.786890 521 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 23cf772ace1c44ecbd450e0bce820c3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:03.819283 521 tablet_copy_client.cc:783] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:03.819695 521 tablet_copy_client.cc:647] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:03.823196 521 tablet_copy_client.cc:523] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
W20240502 14:15:03.824848 31027 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 60030.6us
I20240502 14:15:03.826887 521 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:15:04.066987 557 ts_tablet_manager.cc:902] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
W20240502 14:15:04.169118 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 101223us
I20240502 14:15:04.182734 520 tablet_copy_client.cc:308] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:41511
I20240502 14:15:04.217773 566 ts_tablet_manager.cc:902] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:15:04.243257 566 tablet_copy_client.cc:308] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:41511
W20240502 14:15:04.258275 31027 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 52227.9us
I20240502 14:15:04.275586 557 tablet_copy_client.cc:308] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:41511
I20240502 14:15:04.342908 30984 tablet_copy_service.cc:140] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Received BeginTabletCopySession request for tablet 74c46ec356414d26987e1cd6a0857602 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:51195)
I20240502 14:15:04.343160 30984 tablet_copy_service.cc:161] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Beginning new tablet copy session on tablet 74c46ec356414d26987e1cd6a0857602 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:51195: session id = 3eafb09239024de3bc0eea8d700f7668-74c46ec356414d26987e1cd6a0857602
I20240502 14:15:04.347151 30984 tablet_copy_source_session.cc:205] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:04.348802 30984 tablet_copy_service.cc:140] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Received BeginTabletCopySession request for tablet 50f7b19e515e497c9a27e8e94cb73079 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:51195)
I20240502 14:15:04.349006 30984 tablet_copy_service.cc:161] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Beginning new tablet copy session on tablet 50f7b19e515e497c9a27e8e94cb73079 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:51195: session id = 3eafb09239024de3bc0eea8d700f7668-50f7b19e515e497c9a27e8e94cb73079
I20240502 14:15:04.351694 30984 tablet_copy_source_session.cc:205] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:04.353446 520 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 74c46ec356414d26987e1cd6a0857602. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:04.383373 520 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.StartTabletCopy from 127.27.220.132:56155 (request call id 1) took 1033 ms. Trace:
I20240502 14:15:04.406760 520 rpcz_store.cc:276] 0502 14:15:03.350305 (+ 0us) service_pool.cc:168] Inserting onto call queue
0502 14:15:03.350359 (+ 54us) service_pool.cc:225] Handling call
0502 14:15:03.478233 (+127874us) ts_tablet_manager.cc:903] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
0502 14:15:04.352920 (+874687us) tablet_copy_client.cc:353] Tablet copy session begun
0502 14:15:04.357492 (+ 4572us) tablet_metadata.cc:641] Metadata flushed
0502 14:15:04.357618 (+ 126us) tablet_copy_client.cc:482] Wrote new tablet metadata
0502 14:15:04.361289 (+ 3671us) tablet_copy_client.cc:403] Wrote new consensus metadata
0502 14:15:04.383212 (+ 21923us) tablet_replica.cc:205] Creating consensus instance
0502 14:15:04.383317 (+ 105us) ts_tablet_manager.cc:958] Replica registered.
0502 14:15:04.383336 (+ 19us) inbound_call.cc:173] Queueing success response
Metrics: {"dns_us":82,"tablet-copy.queue_time_us":127452,"thread_start_us":748613,"threads_started":5}
I20240502 14:15:04.352435 30983 tablet_copy_service.cc:140] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Received BeginTabletCopySession request for tablet 5c4ceedcb00a49e1a32977e4e1acea24 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:51195)
I20240502 14:15:04.417070 30983 tablet_copy_service.cc:161] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Beginning new tablet copy session on tablet 5c4ceedcb00a49e1a32977e4e1acea24 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:51195: session id = 3eafb09239024de3bc0eea8d700f7668-5c4ceedcb00a49e1a32977e4e1acea24
I20240502 14:15:04.419912 30983 tablet_copy_source_session.cc:205] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:04.423888 520 tablet_copy_client.cc:783] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:04.425539 520 tablet_copy_client.cc:647] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:04.432610 557 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5c4ceedcb00a49e1a32977e4e1acea24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:04.440779 566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f7b19e515e497c9a27e8e94cb73079. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:04.456238 566 tablet_copy_client.cc:783] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:04.456596 566 tablet_copy_client.cc:647] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:04.484648 557 tablet_copy_client.cc:783] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:04.485054 557 tablet_copy_client.cc:647] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:04.487339 566 tablet_copy_client.cc:523] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:04.489454 520 tablet_copy_client.cc:523] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:04.491025 566 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:15:04.522548 520 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:15:04.536162 557 tablet_copy_client.cc:523] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:04.541720 557 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:15:04.630681 521 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=79 overwritten=0 applied=79 ignored=0} inserts{seen=764 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:04.637912 521 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:15:04.638304 521 ts_tablet_manager.cc:1367] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.812s user 0.137s sys 0.006s
I20240502 14:15:04.639091 521 raft_consensus.cc:348] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:04.645619 521 raft_consensus.cc:729] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:15:04.645922 521 consensus_queue.cc:260] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 79, Last appended: 1.79, Last appended by leader: 79, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 75 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:04.661924 521 ts_tablet_manager.cc:1398] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.023s user 0.005s sys 0.000s
I20240502 14:15:04.898983 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-23cf772ace1c44ecbd450e0bce820c3f received from {username='slave'} at 127.27.220.135:57521
I20240502 14:15:04.899184 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-23cf772ace1c44ecbd450e0bce820c3f on tablet 23cf772ace1c44ecbd450e0bce820c3f with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:15:04.904857 32156 raft_consensus.cc:1203] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.74->[1.75-1.82] Dedup: 1.79->[1.80-1.82]
I20240502 14:15:05.106062 31222 consensus_queue.cc:237] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 104, Committed index: 104, Last appended: 1.104, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:05.108621 30959 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
I20240502 14:15:05.109408 478 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 104, Time since last communication: 0.000s
I20240502 14:15:05.111599 30696 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
W20240502 14:15:05.111965 31157 consensus_peers.cc:485] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bfa09c15ce944ce29fce3bbc1ad4df56. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:05.113677 372 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 104, Time since last communication: 0.000s
I20240502 14:15:05.117205 478 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.105: config changed from index -1 to 105, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:05.119060 30573 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
I20240502 14:15:05.119593 512 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 104, Time since last communication: 0.000s
I20240502 14:15:05.121254 30574 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index -1 to 105, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:05.122669 30959 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index -1 to 105, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:05.127435 30701 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index -1 to 105, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:05.131773 30427 catalog_manager.cc:5484] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index -1 to 105, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:05.156078 478 raft_consensus.cc:1050] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:15:05.157967 478 consensus_queue.cc:237] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 83, Committed index: 83, Last appended: 1.83, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 84 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:05.161271 31092 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 83. Preceding OpId from leader: term: 1 index: 84. (index mismatch)
I20240502 14:15:05.162312 372 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 84, Last known committed idx: 83, Time since last communication: 0.000s
I20240502 14:15:05.169764 32156 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 83. Preceding OpId from leader: term: 1 index: 84. (index mismatch)
I20240502 14:15:05.171486 30700 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 83. Preceding OpId from leader: term: 1 index: 84. (index mismatch)
I20240502 14:15:05.175679 30573 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 83. Preceding OpId from leader: term: 1 index: 84. (index mismatch)
I20240502 14:15:05.177964 372 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 84, Last known committed idx: 83, Time since last communication: 0.000s
I20240502 14:15:05.179318 512 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 84, Last known committed idx: 83, Time since last communication: 0.000s
I20240502 14:15:05.181794 32454 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 84, Last known committed idx: 83, Time since last communication: 0.000s
I20240502 14:15:05.192579 478 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.84: config changed from index 75 to 84, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 84 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:05.198269 31092 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.84: config changed from index 75 to 84, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 84 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:05.207453 30428 catalog_manager.cc:5484] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 75 to 84, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 84 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:05.203830 30573 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.84: config changed from index 75 to 84, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 84 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:05.216627 32156 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.84: config changed from index 75 to 84, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 84 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:05.225019 30696 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.84: config changed from index 75 to 84, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 84 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:05.323179 30959 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 83. Preceding OpId from leader: term: 1 index: 84. (index mismatch)
I20240502 14:15:05.324054 478 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 84, Last known committed idx: 82, Time since last communication: 0.000s
I20240502 14:15:05.329210 30959 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.84: config changed from index 75 to 84, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 84 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:15:05.346326 30767 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 243265us
I20240502 14:15:05.347731 30827 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 104. Preceding OpId from leader: term: 1 index: 105. (index mismatch)
I20240502 14:15:05.400166 478 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 105, Last known committed idx: 104, Time since last communication: 0.000s
I20240502 14:15:05.407675 566 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=112 overwritten=0 applied=112 ignored=0} inserts{seen=797 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:05.415366 566 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:15:05.415800 566 ts_tablet_manager.cc:1367] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.925s user 0.160s sys 0.020s
I20240502 14:15:05.414846 557 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=202 overwritten=0 applied=202 ignored=0} inserts{seen=833 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:05.418567 557 tablet_bootstrap.cc:492] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:15:05.420303 557 ts_tablet_manager.cc:1367] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.879s user 0.262s sys 0.025s
I20240502 14:15:05.419129 566 raft_consensus.cc:348] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:05.420971 566 raft_consensus.cc:729] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:15:05.421293 566 consensus_queue.cc:260] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 112, Last appended: 1.112, Last appended by leader: 112, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 106 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:05.426749 566 ts_tablet_manager.cc:1398] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.010s user 0.002s sys 0.000s
I20240502 14:15:05.422323 557 raft_consensus.cc:348] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:05.427402 557 raft_consensus.cc:729] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:15:05.427733 557 consensus_queue.cc:260] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 202, Last appended: 1.202, Last appended by leader: 202, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:05.428709 557 ts_tablet_manager.cc:1398] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
I20240502 14:15:05.438326 30983 tablet_copy_service.cc:342] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-50f7b19e515e497c9a27e8e94cb73079 received from {username='slave'} at 127.27.220.135:51195
W20240502 14:15:05.438385 30767 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 44192.3us
I20240502 14:15:05.438506 30983 tablet_copy_service.cc:433] P 7c1f568f8efc4b2a9dbf38907f02f3e3: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-50f7b19e515e497c9a27e8e94cb73079 on tablet 50f7b19e515e497c9a27e8e94cb73079 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:15:05.439716 30983 tablet_copy_service.cc:342] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-5c4ceedcb00a49e1a32977e4e1acea24 received from {username='slave'} at 127.27.220.135:51195
I20240502 14:15:05.439846 30983 tablet_copy_service.cc:433] P 7c1f568f8efc4b2a9dbf38907f02f3e3: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-5c4ceedcb00a49e1a32977e4e1acea24 on tablet 5c4ceedcb00a49e1a32977e4e1acea24 with peer 3eafb09239024de3bc0eea8d700f7668
W20240502 14:15:05.498528 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 74533.1us
I20240502 14:15:05.517192 30833 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.105: config changed from index -1 to 105, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:05.564038 557 ts_tablet_manager.cc:902] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:15:05.598367 557 tablet_copy_client.cc:308] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:15:05.694967 31243 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet bfa09c15ce944ce29fce3bbc1ad4df56 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:57521)
I20240502 14:15:05.695181 31243 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet bfa09c15ce944ce29fce3bbc1ad4df56 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:57521: session id = 3eafb09239024de3bc0eea8d700f7668-bfa09c15ce944ce29fce3bbc1ad4df56
I20240502 14:15:05.698235 31243 tablet_copy_source_session.cc:205] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:05.717697 520 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=188 overwritten=0 applied=188 ignored=0} inserts{seen=873 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:05.719524 520 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:15:05.720388 520 ts_tablet_manager.cc:1367] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 1.216s user 0.225s sys 0.051s
I20240502 14:15:05.726461 520 raft_consensus.cc:348] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:05.726864 520 raft_consensus.cc:729] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:15:05.727120 520 consensus_queue.cc:260] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 188, Last appended: 1.188, Last appended by leader: 188, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 164 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:05.727931 520 ts_tablet_manager.cc:1398] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 14:15:05.728682 32158 raft_consensus.cc:1203] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.105->[1.106-1.123] Dedup: 1.112->[1.113-1.123]
I20240502 14:15:05.734436 30983 tablet_copy_service.cc:342] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-74c46ec356414d26987e1cd6a0857602 received from {username='slave'} at 127.27.220.135:51195
I20240502 14:15:05.734675 30983 tablet_copy_service.cc:433] P 7c1f568f8efc4b2a9dbf38907f02f3e3: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-74c46ec356414d26987e1cd6a0857602 on tablet 74c46ec356414d26987e1cd6a0857602 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:15:05.756389 557 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bfa09c15ce944ce29fce3bbc1ad4df56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:05.762964 557 tablet_copy_client.cc:783] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:05.763355 557 tablet_copy_client.cc:647] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:05.903326 32158 raft_consensus.cc:1203] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.163->[1.164-1.204] Dedup: 1.188->[1.189-1.204]
W20240502 14:15:05.913573 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 75032.2us
I20240502 14:15:06.000520 557 tablet_copy_client.cc:523] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:06.004405 557 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:15:06.023777 32157 raft_consensus.cc:1203] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.190->[1.191-1.219] Dedup: 1.202->[1.203-1.219]
I20240502 14:15:06.088723 31222 consensus_queue.cc:237] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 87, Committed index: 87, Last appended: 1.87, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 88 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:06.094244 30571 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 87. Preceding OpId from leader: term: 1 index: 88. (index mismatch)
I20240502 14:15:06.094969 31091 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 87. Preceding OpId from leader: term: 1 index: 88. (index mismatch)
I20240502 14:15:06.094913 478 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 88, Last known committed idx: 87, Time since last communication: 0.000s
I20240502 14:15:06.095609 495 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 88, Last known committed idx: 87, Time since last communication: 0.000s
I20240502 14:15:06.099956 495 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.88: config changed from index 84 to 88, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 88 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.101266 32153 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 87. Preceding OpId from leader: term: 1 index: 88. (index mismatch)
I20240502 14:15:06.103482 30703 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 87. Preceding OpId from leader: term: 1 index: 88. (index mismatch)
I20240502 14:15:06.104343 372 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 88, Last known committed idx: 87, Time since last communication: 0.000s
I20240502 14:15:06.115909 30571 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.88: config changed from index 84 to 88, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 88 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.116909 512 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 88, Last known committed idx: 87, Time since last communication: 0.000s
I20240502 14:15:06.118580 31091 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.88: config changed from index 84 to 88, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 88 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.126849 30427 catalog_manager.cc:5484] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 84 to 88, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 88 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.142212 621 raft_consensus.cc:1050] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:15:06.145978 621 consensus_queue.cc:237] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 124, Committed index: 124, Last appended: 1.124, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 125 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:06.149279 31092 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 124. Preceding OpId from leader: term: 1 index: 125. (index mismatch)
I20240502 14:15:06.149830 346 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 125, Last known committed idx: 124, Time since last communication: 0.000s
I20240502 14:15:06.153579 31222 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 124. Preceding OpId from leader: term: 1 index: 125. (index mismatch)
I20240502 14:15:06.154722 415 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 125, Last known committed idx: 124, Time since last communication: 0.000s
I20240502 14:15:06.161267 30831 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 124. Preceding OpId from leader: term: 1 index: 125. (index mismatch)
I20240502 14:15:06.164801 32158 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 124. Preceding OpId from leader: term: 1 index: 125. (index mismatch)
I20240502 14:15:06.164950 30574 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 124. Preceding OpId from leader: term: 1 index: 125. (index mismatch)
I20240502 14:15:06.165823 32600 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 125, Last known committed idx: 124, Time since last communication: 0.000s
I20240502 14:15:06.168752 519 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.125: config changed from index 106 to 125, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 125 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.175078 31222 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.125: config changed from index 106 to 125, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 125 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.175601 30571 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.125: config changed from index 106 to 125, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 125 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.178910 31092 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.125: config changed from index 106 to 125, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 125 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.181270 346 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 125, Last known committed idx: 123, Time since last communication: 0.000s
I20240502 14:15:06.192574 32155 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.88: config changed from index 84 to 88, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 88 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.287101 519 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 125, Last known committed idx: 124, Time since last communication: 0.000s
I20240502 14:15:06.289616 30831 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.125: config changed from index 106 to 125, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 125 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:15:06.294116 660 log.cc:924] Time spent T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668: Append to log took a long time: real 0.178s user 0.000s sys 0.000s
I20240502 14:15:06.298457 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 23cf772ace1c44ecbd450e0bce820c3f with cas_config_opid_index 84: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:06.303927 30699 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.88: config changed from index 84 to 88, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 88 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.331099 621 raft_consensus.cc:1050] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:15:06.332412 621 consensus_queue.cc:237] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 211, Committed index: 211, Last appended: 1.211, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 212 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:06.335151 31091 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 211. Preceding OpId from leader: term: 1 index: 212. (index mismatch)
I20240502 14:15:06.335575 30573 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 211. Preceding OpId from leader: term: 1 index: 212. (index mismatch)
I20240502 14:15:06.335965 635 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 212, Last known committed idx: 211, Time since last communication: 0.000s
I20240502 14:15:06.336383 635 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 212, Last known committed idx: 211, Time since last communication: 0.000s
I20240502 14:15:06.342417 30827 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 211. Preceding OpId from leader: term: 1 index: 212. (index mismatch)
I20240502 14:15:06.343065 32157 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.125: config changed from index 106 to 125, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 125 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.360473 32157 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 210. Preceding OpId from leader: term: 1 index: 212. (index mismatch)
I20240502 14:15:06.360548 30701 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 211. Preceding OpId from leader: term: 1 index: 212. (index mismatch)
I20240502 14:15:06.395821 30428 catalog_manager.cc:5484] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 106 to 125, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 125 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.399356 635 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 212, Last known committed idx: 211, Time since last communication: 0.000s
I20240502 14:15:06.413583 415 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 212, Last known committed idx: 210, Time since last communication: 0.000s
I20240502 14:15:06.440424 30943 tablet_service.cc:1514] Processing DeleteTablet for tablet 23cf772ace1c44ecbd450e0bce820c3f with delete_type TABLET_DATA_TOMBSTONED (TS 7c1f568f8efc4b2a9dbf38907f02f3e3 not found in new config with opid_index 88) from {username='slave'} at 127.0.0.1:53120
I20240502 14:15:06.447986 674 tablet_replica.cc:330] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: stopping tablet replica
I20240502 14:15:06.448233 674 raft_consensus.cc:2229] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:06.448508 674 raft_consensus.cc:2258] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:06.449645 674 ts_tablet_manager.cc:1875] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:06.450112 415 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 212, Last known committed idx: 210, Time since last communication: 0.000s
I20240502 14:15:06.454298 635 raft_consensus.cc:1050] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:15:06.456003 635 consensus_queue.cc:237] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 234, Committed index: 234, Last appended: 1.234, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:06.458504 30574 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 234. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 14:15:06.458526 31222 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 234. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 14:15:06.459139 312 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 235, Last known committed idx: 234, Time since last communication: 0.000s
I20240502 14:15:06.459568 519 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 235, Last known committed idx: 234, Time since last communication: 0.000s
I20240502 14:15:06.467031 557 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=109 overwritten=0 applied=109 ignored=0} inserts{seen=740 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:06.467568 557 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:15:06.467985 557 ts_tablet_manager.cc:1367] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.464s user 0.146s sys 0.034s
I20240502 14:15:06.468878 557 raft_consensus.cc:348] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:06.469260 557 raft_consensus.cc:729] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:15:06.469558 557 consensus_queue.cc:260] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 109, Last appended: 1.109, Last appended by leader: 109, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 105 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:06.469976 674 ts_tablet_manager.cc:1888] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.87
I20240502 14:15:06.470278 674 log.cc:1196] T 23cf772ace1c44ecbd450e0bce820c3f P 7c1f568f8efc4b2a9dbf38907f02f3e3: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/23cf772ace1c44ecbd450e0bce820c3f
I20240502 14:15:06.470350 557 ts_tablet_manager.cc:1398] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:15:06.475528 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-bfa09c15ce944ce29fce3bbc1ad4df56 received from {username='slave'} at 127.27.220.135:57521
I20240502 14:15:06.475682 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-bfa09c15ce944ce29fce3bbc1ad4df56 on tablet bfa09c15ce944ce29fce3bbc1ad4df56 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:15:06.476655 30418 catalog_manager.cc:4830] TS 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): tablet 23cf772ace1c44ecbd450e0bce820c3f (table rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]) successfully deleted
I20240502 14:15:06.478025 32157 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 234. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 14:15:06.479671 312 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.212: config changed from index 164 to 212, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 212 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.508961 30833 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 234. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 14:15:06.511518 30699 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 234. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 14:15:06.556648 30573 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.212: config changed from index 164 to 212, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 212 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.563694 30699 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.212: config changed from index 164 to 212, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 212 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.565963 635 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 235, Last known committed idx: 234, Time since last communication: 0.000s
I20240502 14:15:06.567164 635 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 235, Last known committed idx: 234, Time since last communication: 0.000s
I20240502 14:15:06.566915 30827 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.212: config changed from index 164 to 212, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 212 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.570812 635 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 235, Last known committed idx: 234, Time since last communication: 0.000s
I20240502 14:15:06.574961 30427 catalog_manager.cc:5484] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 164 to 212, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 212 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.596205 32155 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.212: config changed from index 164 to 212, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 212 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:15:06.607028 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 59317us
I20240502 14:15:06.608256 32155 raft_consensus.cc:1203] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.104->[1.105-1.113] Dedup: 1.109->[1.110-1.113]
I20240502 14:15:06.610070 31091 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.212: config changed from index 164 to 212, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 212 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:15:06.693498 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 80032.1us
I20240502 14:15:06.699252 415 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.235: config changed from index 191 to 235, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.707962 31222 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.235: config changed from index 191 to 235, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.723363 30703 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.235: config changed from index 191 to 235, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.733224 30574 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.235: config changed from index 191 to 235, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.735178 30433 catalog_manager.cc:5484] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 reported cstate change: config changed from index 191 to 235, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.747961 30831 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.235: config changed from index 191 to 235, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.780704 30964 consensus_queue.cc:237] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 128, Committed index: 128, Last appended: 1.128, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:06.789317 30569 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 128. Preceding OpId from leader: term: 1 index: 129. (index mismatch)
I20240502 14:15:06.789333 31092 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 128. Preceding OpId from leader: term: 1 index: 129. (index mismatch)
I20240502 14:15:06.790026 621 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 129, Last known committed idx: 128, Time since last communication: 0.000s
I20240502 14:15:06.790550 621 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 129, Last known committed idx: 128, Time since last communication: 0.000s
I20240502 14:15:06.796432 415 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.129: config changed from index 125 to 129, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.797808 30574 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.129: config changed from index 125 to 129, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.800498 31224 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 128. Preceding OpId from leader: term: 1 index: 129. (index mismatch)
I20240502 14:15:06.801702 519 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 129, Last known committed idx: 128, Time since last communication: 0.000s
I20240502 14:15:06.806660 31224 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.129: config changed from index 125 to 129, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.811313 30433 catalog_manager.cc:5484] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 125 to 129, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.814348 31094 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.129: config changed from index 125 to 129, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.816323 30813 tablet_service.cc:1514] Processing DeleteTablet for tablet 50f7b19e515e497c9a27e8e94cb73079 with delete_type TABLET_DATA_TOMBSTONED (TS 155f26c73dc84441a58417c09f577494 not found in new config with opid_index 129) from {username='slave'} at 127.0.0.1:33764
I20240502 14:15:06.822870 32157 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.235: config changed from index 191 to 235, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 235 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.825814 32156 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 128. Preceding OpId from leader: term: 1 index: 129. (index mismatch)
I20240502 14:15:06.836758 30418 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 50f7b19e515e497c9a27e8e94cb73079 with cas_config_opid_index 125: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:06.838951 30961 consensus_queue.cc:237] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 235, Committed index: 235, Last appended: 1.235, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:06.841363 30964 consensus_queue.cc:237] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 214, Committed index: 214, Last appended: 1.214, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 215 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:06.841627 31222 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 235. Preceding OpId from leader: term: 1 index: 236. (index mismatch)
I20240502 14:15:06.859822 621 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 236, Last known committed idx: 235, Time since last communication: 0.007s
I20240502 14:15:06.872279 32157 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 235. Preceding OpId from leader: term: 1 index: 236. (index mismatch)
I20240502 14:15:06.872409 30831 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 235. Preceding OpId from leader: term: 1 index: 236. (index mismatch)
I20240502 14:15:06.874353 30831 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 214. Preceding OpId from leader: term: 1 index: 215. (index mismatch)
I20240502 14:15:06.874421 30703 raft_consensus.cc:1261] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 235. Preceding OpId from leader: term: 1 index: 236. (index mismatch)
I20240502 14:15:06.876080 32158 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 214. Preceding OpId from leader: term: 1 index: 215. (index mismatch)
I20240502 14:15:06.886502 31094 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 214. Preceding OpId from leader: term: 1 index: 215. (index mismatch)
I20240502 14:15:06.894961 30569 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 214. Preceding OpId from leader: term: 1 index: 215. (index mismatch)
I20240502 14:15:06.895264 519 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 236, Last known committed idx: 235, Time since last communication: 0.000s
I20240502 14:15:06.896654 32600 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 215, Last known committed idx: 214, Time since last communication: 0.000s
I20240502 14:15:06.911623 519 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 215, Last known committed idx: 214, Time since last communication: 0.000s
I20240502 14:15:06.912276 519 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 129, Last known committed idx: 128, Time since last communication: 0.000s
I20240502 14:15:06.912880 519 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 236, Last known committed idx: 235, Time since last communication: 0.000s
I20240502 14:15:06.913753 519 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 215, Last known committed idx: 213, Time since last communication: 0.000s
I20240502 14:15:06.919849 703 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 215, Last known committed idx: 214, Time since last communication: 0.000s
I20240502 14:15:06.923854 621 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.215: config changed from index 212 to 215, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 215 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.935076 30569 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.215: config changed from index 212 to 215, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 215 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.941625 346 consensus_queue.cc:1035] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 236, Last known committed idx: 235, Time since last communication: 0.000s
I20240502 14:15:06.945168 31089 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.215: config changed from index 212 to 215, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 215 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.945326 30433 catalog_manager.cc:5484] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 212 to 215, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 215 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.953938 30682 tablet_service.cc:1514] Processing DeleteTablet for tablet 74c46ec356414d26987e1cd6a0857602 with delete_type TABLET_DATA_TOMBSTONED (TS d733c51902044956a8cd2b6b31bc9831 not found in new config with opid_index 215) from {username='slave'} at 127.0.0.1:39850
I20240502 14:15:06.937150 701 tablet_replica.cc:330] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: stopping tablet replica
I20240502 14:15:06.960789 701 raft_consensus.cc:2229] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:06.961750 495 raft_consensus.cc:1050] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:15:06.962338 701 raft_consensus.cc:2258] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:06.963881 701 ts_tablet_manager.cc:1875] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:06.966179 495 consensus_queue.cc:237] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 114, Committed index: 114, Last appended: 1.114, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:06.970005 30569 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 114. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
I20240502 14:15:06.971369 478 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 114, Time since last communication: 0.000s
I20240502 14:15:06.981393 32158 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.129: config changed from index 125 to 129, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 129 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:06.991130 701 ts_tablet_manager.cc:1888] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.128
I20240502 14:15:06.991359 701 log.cc:1196] T 50f7b19e515e497c9a27e8e94cb73079 P 155f26c73dc84441a58417c09f577494: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/50f7b19e515e497c9a27e8e94cb73079
I20240502 14:15:06.995381 30417 catalog_manager.cc:4830] TS 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): tablet 50f7b19e515e497c9a27e8e94cb73079 (table rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]) successfully deleted
I20240502 14:15:06.996073 30827 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 114. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
I20240502 14:15:07.004182 30699 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 114. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
I20240502 14:15:07.004887 30418 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 74c46ec356414d26987e1cd6a0857602 with cas_config_opid_index 212: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:07.005888 32157 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 114. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
I20240502 14:15:07.013800 478 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 114, Time since last communication: 0.000s
I20240502 14:15:07.014070 346 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.236: config changed from index 235 to 236, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.019564 31222 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.236: config changed from index 235 to 236, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.025602 663 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 114, Time since last communication: 0.000s
I20240502 14:15:07.029883 30961 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 114. Preceding OpId from leader: term: 1 index: 115. (index mismatch)
I20240502 14:15:07.039810 32157 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.236: config changed from index 235 to 236, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.063386 30427 catalog_manager.cc:5484] T 5c4ceedcb00a49e1a32977e4e1acea24 P 3eafb09239024de3bc0eea8d700f7668 reported cstate change: config changed from index 235 to 236, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.070662 30553 tablet_service.cc:1514] Processing DeleteTablet for tablet 5c4ceedcb00a49e1a32977e4e1acea24 with delete_type TABLET_DATA_TOMBSTONED (TS fd3399b1bfe644159a833ede6726c25f not found in new config with opid_index 236) from {username='slave'} at 127.0.0.1:38096
I20240502 14:15:07.074420 706 tablet_replica.cc:330] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: stopping tablet replica
I20240502 14:15:07.077176 32157 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.215: config changed from index 212 to 215, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 215 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.078500 713 tablet_replica.cc:330] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: stopping tablet replica
I20240502 14:15:07.078820 713 raft_consensus.cc:2229] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:07.080910 713 raft_consensus.cc:2258] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:07.081912 30418 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5c4ceedcb00a49e1a32977e4e1acea24 with cas_config_opid_index 235: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:07.082659 30827 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.236: config changed from index 235 to 236, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.088851 706 raft_consensus.cc:2229] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:07.089345 663 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 114, Time since last communication: 0.000s
I20240502 14:15:07.091208 706 raft_consensus.cc:2258] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:07.094192 706 ts_tablet_manager.cc:1875] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:07.094954 512 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 115, Last known committed idx: 114, Time since last communication: 0.000s
I20240502 14:15:07.102277 663 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.115: config changed from index 105 to 115, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.102864 30831 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.215: config changed from index 212 to 215, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 215 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.108361 30569 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index 105 to 115, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.109560 30696 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index 105 to 115, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.114449 30701 raft_consensus.cc:2932] T 5c4ceedcb00a49e1a32977e4e1acea24 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.236: config changed from index 235 to 236, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 236 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.129923 30426 catalog_manager.cc:5484] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 105 to 115, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.132908 713 ts_tablet_manager.cc:1875] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:07.132503 32156 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index 105 to 115, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.137480 30832 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index 105 to 115, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.152899 706 ts_tablet_manager.cc:1888] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.214
I20240502 14:15:07.156183 706 log.cc:1196] T 74c46ec356414d26987e1cd6a0857602 P d733c51902044956a8cd2b6b31bc9831: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/74c46ec356414d26987e1cd6a0857602
I20240502 14:15:07.177353 30417 catalog_manager.cc:4830] TS d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): tablet 74c46ec356414d26987e1cd6a0857602 (table rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]) successfully deleted
I20240502 14:15:07.230239 31222 consensus_queue.cc:237] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 116, Committed index: 116, Last appended: 1.117, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 118 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:07.236780 30960 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.115: config changed from index 105 to 115, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 115 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.255040 30569 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 117. Preceding OpId from leader: term: 1 index: 118. (index mismatch)
I20240502 14:15:07.255692 495 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 118, Last known committed idx: 116, Time since last communication: 0.000s
I20240502 14:15:07.256258 713 ts_tablet_manager.cc:1888] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.235
I20240502 14:15:07.261283 32157 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 117. Preceding OpId from leader: term: 1 index: 118. (index mismatch)
I20240502 14:15:07.264053 30961 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 117. Preceding OpId from leader: term: 1 index: 118. (index mismatch)
I20240502 14:15:07.264393 713 log.cc:1196] T 5c4ceedcb00a49e1a32977e4e1acea24 P fd3399b1bfe644159a833ede6726c25f: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/5c4ceedcb00a49e1a32977e4e1acea24
I20240502 14:15:07.266027 30827 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 117. Preceding OpId from leader: term: 1 index: 118. (index mismatch)
I20240502 14:15:07.266158 478 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 118, Last known committed idx: 116, Time since last communication: 0.000s
I20240502 14:15:07.266548 30419 catalog_manager.cc:4830] TS fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): tablet 5c4ceedcb00a49e1a32977e4e1acea24 (table rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]) successfully deleted
I20240502 14:15:07.273989 478 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.118: config changed from index 115 to 118, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 118 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.285137 30960 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.118: config changed from index 115 to 118, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 118 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.287840 30573 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.118: config changed from index 115 to 118, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 118 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.298616 32454 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 118, Last known committed idx: 116, Time since last communication: 0.000s
I20240502 14:15:07.298529 30433 catalog_manager.cc:5484] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index 115 to 118, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 118 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.299893 32454 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 118, Last known committed idx: 116, Time since last communication: 0.000s
I20240502 14:15:07.304720 30682 tablet_service.cc:1514] Processing DeleteTablet for tablet bfa09c15ce944ce29fce3bbc1ad4df56 with delete_type TABLET_DATA_TOMBSTONED (TS d733c51902044956a8cd2b6b31bc9831 not found in new config with opid_index 118) from {username='slave'} at 127.0.0.1:39850
I20240502 14:15:07.305076 706 tablet_replica.cc:330] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: stopping tablet replica
I20240502 14:15:07.305287 706 raft_consensus.cc:2229] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:07.305462 706 pending_rounds.cc:62] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: Trying to abort 1 pending ops.
I20240502 14:15:07.305596 706 pending_rounds.cc:68] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: Aborting op as it isn't in flight: id { term: 1 index: 117 } timestamp: 7023244522197028864 op_type: WRITE_OP write_request { tablet_id: "bfa09c15ce944ce29fce3bbc1ad4df56" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000d\317\274H9<\345g\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""99e1932451065b84" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023244521545486336 authz_token { token_data: "\010\204\302\316\261\006\"3\n\005slave\022*\n 524e8e464d784197b8e14eac16bb56bb\020\001\030\001 \001(\001" signature: "<redacted>""\265\r\352\363\215\376\324d\r\357\"\256)y\362h\257W~\377\241{\211\027\315yB.&\217;\226\376\242D\322\214\327>t>\027\2371~\352\036\203\004\335JW\345|(\206\304-\270\225\037g\232&" signing_key_seq_num: 0 } } request_id { client_id: "cbc7e379089b4d8fa6b0529d593aba81" seq_no: 319 first_incomplete_seq_no: 319 attempt_no: 0 }
I20240502 14:15:07.306262 706 raft_consensus.cc:2258] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:07.321928 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bfa09c15ce944ce29fce3bbc1ad4df56 with cas_config_opid_index 115: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:07.326865 30829 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.118: config changed from index 115 to 118, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 118 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:07.327700 706 ts_tablet_manager.cc:1875] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:07.333915 706 ts_tablet_manager.cc:1888] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.117
I20240502 14:15:07.334132 706 log.cc:1196] T bfa09c15ce944ce29fce3bbc1ad4df56 P d733c51902044956a8cd2b6b31bc9831: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/bfa09c15ce944ce29fce3bbc1ad4df56
I20240502 14:15:07.335196 30417 catalog_manager.cc:4830] TS d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): tablet bfa09c15ce944ce29fce3bbc1ad4df56 (table rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]) successfully deleted
I20240502 14:15:07.339102 32155 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.118: config changed from index 115 to 118, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 118 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:15:07.389592 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 45945.6us
W20240502 14:15:07.513655 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 49691.3us
W20240502 14:15:07.600251 731 log.cc:924] Time spent T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849: Append to log took a long time: real 0.055s user 0.000s sys 0.000s
W20240502 14:15:07.669859 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 71477.4us
I20240502 14:15:08.017684 30573 consensus_queue.cc:237] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:08.174610 30960 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:15:08.175408 678 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 14:15:08.194799 31223 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:15:08.217712 678 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 14:15:08.227371 31094 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:15:08.256513 678 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 14:15:08.334182 30832 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:15:08.335531 678 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 14:15:08.360611 728 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:08.374380 30433 catalog_manager.cc:5484] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index -1 to 2, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:15:08.374579 30964 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:08.392062 30832 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:08.392562 31222 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
I20240502 14:15:08.504917 31094 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } } }
W20240502 14:15:08.505596 30507 consensus_peers.cc:485] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f -> Peer 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): Couldn't send request to peer 3eafb09239024de3bc0eea8d700f7668. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b63847bdc0e546bca632a85255a45572. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:08.704494 793 ts_tablet_manager.cc:902] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: Initiating tablet copy from peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:15:08.762609 793 tablet_copy_client.cc:308] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:33055
I20240502 14:15:08.788770 30594 tablet_copy_service.cc:140] P fd3399b1bfe644159a833ede6726c25f: Received BeginTabletCopySession request for tablet b63847bdc0e546bca632a85255a45572 from peer 3eafb09239024de3bc0eea8d700f7668 ({username='slave'} at 127.27.220.135:51313)
I20240502 14:15:08.789053 30594 tablet_copy_service.cc:161] P fd3399b1bfe644159a833ede6726c25f: Beginning new tablet copy session on tablet b63847bdc0e546bca632a85255a45572 from peer 3eafb09239024de3bc0eea8d700f7668 at {username='slave'} at 127.27.220.135:51313: session id = 3eafb09239024de3bc0eea8d700f7668-b63847bdc0e546bca632a85255a45572
I20240502 14:15:08.795734 30594 tablet_copy_source_session.cc:205] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:08.798007 793 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b63847bdc0e546bca632a85255a45572. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:08.806862 793 tablet_copy_client.cc:783] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:08.807264 793 tablet_copy_client.cc:647] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:08.811056 793 tablet_copy_client.cc:523] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:08.817764 793 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap starting.
I20240502 14:15:08.857157 793 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:08.857776 793 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: Bootstrap complete.
I20240502 14:15:08.858101 793 ts_tablet_manager.cc:1367] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: Time spent bootstrapping tablet: real 0.041s user 0.006s sys 0.012s
I20240502 14:15:08.859059 793 raft_consensus.cc:348] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:08.859445 793 raft_consensus.cc:729] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3eafb09239024de3bc0eea8d700f7668, State: Initialized, Role: LEARNER
I20240502 14:15:08.859692 793 consensus_queue.cc:260] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: true } }
I20240502 14:15:08.867009 793 ts_tablet_manager.cc:1398] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668: Time spent starting tablet: real 0.009s user 0.006s sys 0.002s
I20240502 14:15:08.868085 30594 tablet_copy_service.cc:342] P fd3399b1bfe644159a833ede6726c25f: Request end of tablet copy session 3eafb09239024de3bc0eea8d700f7668-b63847bdc0e546bca632a85255a45572 received from {username='slave'} at 127.27.220.135:51313
I20240502 14:15:08.868336 30594 tablet_copy_service.cc:433] P fd3399b1bfe644159a833ede6726c25f: ending tablet copy session 3eafb09239024de3bc0eea8d700f7668-b63847bdc0e546bca632a85255a45572 on tablet b63847bdc0e546bca632a85255a45572 with peer 3eafb09239024de3bc0eea8d700f7668
I20240502 14:15:09.278659 32155 raft_consensus.cc:1203] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 14:15:09.718042 728 raft_consensus.cc:1050] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: attempting to promote NON_VOTER 3eafb09239024de3bc0eea8d700f7668 to VOTER
I20240502 14:15:09.719338 728 consensus_queue.cc:237] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:09.722829 32155 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 LEARNER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 14:15:09.724000 776 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 14:15:09.729198 30959 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 14:15:09.729915 30831 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 14:15:09.730847 506 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 14:15:09.733243 776 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 14:15:09.749109 31089 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 14:15:09.752293 728 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 14:15:09.764494 31222 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 14:15:09.775986 728 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 14:15:09.799592 728 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:09.802292 32158 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:09.813241 30827 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:09.806023 30960 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:09.819160 30426 catalog_manager.cc:5484] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 2 to 3, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:09.879542 31092 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:15:09.923499 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 68716.4us
I20240502 14:15:09.967149 30571 consensus_queue.cc:237] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } }
I20240502 14:15:09.973407 30964 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:15:09.974483 678 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:15:09.974323 31222 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:09.978559 31089 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:15:09.984265 728 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:15:09.997350 678 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:10.002739 30419 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b63847bdc0e546bca632a85255a45572 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:10.004431 30427 catalog_manager.cc:5484] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 3 to 4, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:15:10.006986 31089 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:10.021051 32154 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:15:10.022723 728 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:15:10.024230 32158 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:10.029773 30961 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
I20240502 14:15:10.037021 30813 tablet_service.cc:1514] Processing DeleteTablet for tablet b63847bdc0e546bca632a85255a45572 with delete_type TABLET_DATA_TOMBSTONED (TS 155f26c73dc84441a58417c09f577494 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:33764
I20240502 14:15:10.052345 31222 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:15:10.090636 855 tablet_replica.cc:330] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: stopping tablet replica
I20240502 14:15:10.091423 855 raft_consensus.cc:2229] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:10.091801 855 raft_consensus.cc:2258] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:10.098168 855 ts_tablet_manager.cc:1875] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:10.106925 728 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:15:10.138593 855 ts_tablet_manager.cc:1888] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 14:15:10.138760 855 log.cc:1196] T b63847bdc0e546bca632a85255a45572 P 155f26c73dc84441a58417c09f577494: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/b63847bdc0e546bca632a85255a45572
I20240502 14:15:10.139734 30417 catalog_manager.cc:4830] TS 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): tablet b63847bdc0e546bca632a85255a45572 (table TestTable [id=9f6d350f0a5941b88f4dc33c6d359c08]) successfully deleted
I20240502 14:15:10.143958 31224 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } }
W20240502 14:15:10.565922 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:41511, user_credentials={real_user=slave}} blocked reactor thread for 66504.4us
W20240502 14:15:10.728855 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 64170.1us
W20240502 14:15:11.439375 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 268057us
W20240502 14:15:11.966977 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 65626us
I20240502 14:15:12.148064 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:15:12.217234 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:12.283565 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 14:15:12.355039 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 61867.7us
I20240502 14:15:12.807965 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:12.858223 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:12.926769 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:12.961619 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 14:15:13.019837 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 56602.5us
W20240502 14:15:13.421820 30768 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 105676us
W20240502 14:15:13.436071 30769 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.129:33055, user_credentials={real_user=slave}} blocked reactor thread for 144539us
W20240502 14:15:13.552259 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 76538.9us
W20240502 14:15:14.325829 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 61835.9us
I20240502 14:15:14.594218 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:14.629550 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:15:14.711087 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:14.750020 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:14.902518 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:15.056354 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:15.112947 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20240502 14:15:15.677749 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 103005us
W20240502 14:15:16.129783 31027 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 44916.3us
W20240502 14:15:16.303164 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 65216.9us
W20240502 14:15:16.533792 30767 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 44388.7us
W20240502 14:15:16.613049 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:34805, user_credentials={real_user=slave}} blocked reactor thread for 55550.9us
W20240502 14:15:17.125327 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 109764us
W20240502 14:15:17.563361 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:41511, user_credentials={real_user=slave}} blocked reactor thread for 49415.3us
W20240502 14:15:17.620462 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 63111us
W20240502 14:15:17.939486 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 118534us
W20240502 14:15:19.731005 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 119950us
W20240502 14:15:21.461637 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 98330.9us
W20240502 14:15:21.519062 30768 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:41511, user_credentials={real_user=slave}} blocked reactor thread for 62493.2us
I20240502 14:15:21.553470 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:21.554286 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:21.600598 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:21.626210 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:21.685792 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:21.699805 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:15:21.928426 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 14:15:22.025120 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 62282.8us
W20240502 14:15:22.045929 1200 log.cc:924] Time spent T d4aa1eb08d4b450f9c5be499d8c6bbc7 P d733c51902044956a8cd2b6b31bc9831: Append to log took a long time: real 0.096s user 0.000s sys 0.000s
W20240502 14:15:23.577850 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 77381.4us
W20240502 14:15:23.634527 31091 thread.cc:636] prepare [worker] (thread pool) Time spent creating pthread: real 0.683s user 0.000s sys 0.010s
W20240502 14:15:23.634769 31091 thread.cc:602] prepare [worker] (thread pool) Time spent starting thread: real 0.683s user 0.000s sys 0.010s
W20240502 14:15:23.649714 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 47565.1us
W20240502 14:15:24.026160 1201 thread.cc:636] apply [worker] (thread pool) Time spent creating pthread: real 0.829s user 0.020s sys 0.059s
W20240502 14:15:24.026413 1201 thread.cc:602] apply [worker] (thread pool) Time spent starting thread: real 0.830s user 0.020s sys 0.059s
W20240502 14:15:24.027716 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 62997.6us
W20240502 14:15:24.173729 30900 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.134:39403, user_credentials={real_user=slave}} blocked reactor thread for 88170.4us
W20240502 14:15:24.920140 30769 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.129:33055, user_credentials={real_user=slave}} blocked reactor thread for 66372.9us
W20240502 14:15:24.977020 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 46800.4us
W20240502 14:15:25.037580 30768 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:41511, user_credentials={real_user=slave}} blocked reactor thread for 186602us
W20240502 14:15:25.725589 31157 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 55012.7us
I20240502 14:15:25.739845 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:25.745604 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:25.774534 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:25.775781 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:25.804497 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:25.864208 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:15:25.956379 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20240502 14:15:26.001945 30767 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 67222us
W20240502 14:15:26.008371 30897 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:34975, user_credentials={real_user=slave}} blocked reactor thread for 60716.7us
W20240502 14:15:26.488348 30767 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:33091, user_credentials={real_user=slave}} blocked reactor thread for 79261.8us
W20240502 14:15:26.547914 30767 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 45096.7us
I20240502 14:15:27.255606 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:15:27.256374 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:27.279309 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:27.295038 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:27.339310 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:27.365291 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:27.398636 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
1cb98ed728ea452ab7c2d0de3cb89c33 | 127.27.220.190:33157 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+----------------------+-------------------------
builtin_ntp_servers | 127.27.220.148:41401 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+----------------------+---------+----------+----------------+-----------------
0dfd48d0b03a42f689e58d5190d03a2f | 127.27.220.134:39403 | HEALTHY | <none> | 7 | 0
155f26c73dc84441a58417c09f577494 | 127.27.220.131:34975 | HEALTHY | <none> | 2 | 0
3eafb09239024de3bc0eea8d700f7668 | 127.27.220.135:33091 | HEALTHY | <none> | 0 | 0
7c1f568f8efc4b2a9dbf38907f02f3e3 | 127.27.220.132:41511 | HEALTHY | <none> | 3 | 0
ba420f2442d6416bbda95edfff85e849 | 127.27.220.133:34805 | HEALTHY | <none> | 2 | 0
d733c51902044956a8cd2b6b31bc9831 | 127.27.220.130:46259 | HEALTHY | <none> | 0 | 0
fd3399b1bfe644159a833ede6726c25f | 127.27.220.129:33055 | HEALTHY | <none> | 2 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 7
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 7 server(s) checked
local_ip_for_outbound_sockets | 127.27.220.129 | experimental | 127.27.220.129:33055
local_ip_for_outbound_sockets | 127.27.220.130 | experimental | 127.27.220.130:46259
local_ip_for_outbound_sockets | 127.27.220.131 | experimental | 127.27.220.131:34975
local_ip_for_outbound_sockets | 127.27.220.132 | experimental | 127.27.220.132:41511
local_ip_for_outbound_sockets | 127.27.220.133 | experimental | 127.27.220.133:34805
local_ip_for_outbound_sockets | 127.27.220.134 | experimental | 127.27.220.134:39403
local_ip_for_outbound_sockets | 127.27.220.135 | experimental | 127.27.220.135:33091
never_fsync | true | unsafe,advanced | all 7 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 7 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 7 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 7 server(s) checked
server_dump_info_format | pb | hidden | all 7 server(s) checked
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.27.220.129:33055
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.27.220.130:46259
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.27.220.131:34975
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.27.220.132:41511
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.27.220.133:34805
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.27.220.134:39403
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.27.220.135:33091
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+----------------------+-------------------------
builtin_ntp_servers | 127.27.220.148:41401 | all 7 server(s) checked
time_source | builtin | all 7 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 8 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 5 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 11
Median | 11
Third Quartile | 12
Maximum | 12
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 7
Tables | 6
Tablets | 16
Replicas | 80
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 14:15:27.563860 28530 log_verifier.cc:126] Checking tablet 0f906c7d775540e6ba76cf2ff3700a67
I20240502 14:15:27.805881 28530 log_verifier.cc:177] Verified matching terms for 224 ops in tablet 0f906c7d775540e6ba76cf2ff3700a67
I20240502 14:15:27.806500 28530 log_verifier.cc:126] Checking tablet 23cf772ace1c44ecbd450e0bce820c3f
I20240502 14:15:28.032153 28530 log_verifier.cc:177] Verified matching terms for 185 ops in tablet 23cf772ace1c44ecbd450e0bce820c3f
I20240502 14:15:28.032649 28530 log_verifier.cc:126] Checking tablet 3183d887deec4e40b88dceda3306c0f1
I20240502 14:15:28.329499 28530 log_verifier.cc:177] Verified matching terms for 247 ops in tablet 3183d887deec4e40b88dceda3306c0f1
I20240502 14:15:28.330159 28530 log_verifier.cc:126] Checking tablet 356c1606c56147a2868e60ab63c59c13
I20240502 14:15:28.641429 28530 log_verifier.cc:177] Verified matching terms for 292 ops in tablet 356c1606c56147a2868e60ab63c59c13
I20240502 14:15:28.642205 28530 log_verifier.cc:126] Checking tablet 36b27125eebb410e8c37560450778dd5
I20240502 14:15:28.953043 28530 log_verifier.cc:177] Verified matching terms for 264 ops in tablet 36b27125eebb410e8c37560450778dd5
I20240502 14:15:28.953935 28530 log_verifier.cc:126] Checking tablet 3c26d346223a498bb96b962fa7dd99aa
I20240502 14:15:29.244786 28530 log_verifier.cc:177] Verified matching terms for 244 ops in tablet 3c26d346223a498bb96b962fa7dd99aa
I20240502 14:15:29.245527 28530 log_verifier.cc:126] Checking tablet 50f7b19e515e497c9a27e8e94cb73079
I20240502 14:15:29.589468 28530 log_verifier.cc:177] Verified matching terms for 269 ops in tablet 50f7b19e515e497c9a27e8e94cb73079
I20240502 14:15:29.590137 28530 log_verifier.cc:126] Checking tablet 5c4ceedcb00a49e1a32977e4e1acea24
I20240502 14:15:30.018450 28530 log_verifier.cc:177] Verified matching terms for 434 ops in tablet 5c4ceedcb00a49e1a32977e4e1acea24
I20240502 14:15:30.019407 28530 log_verifier.cc:126] Checking tablet 74c46ec356414d26987e1cd6a0857602
I20240502 14:15:30.455158 28530 log_verifier.cc:177] Verified matching terms for 416 ops in tablet 74c46ec356414d26987e1cd6a0857602
I20240502 14:15:30.456169 28530 log_verifier.cc:126] Checking tablet 7e5aa36e1a914374807e91f44bb1cff9
I20240502 14:15:30.776840 28530 log_verifier.cc:177] Verified matching terms for 298 ops in tablet 7e5aa36e1a914374807e91f44bb1cff9
I20240502 14:15:30.777578 28530 log_verifier.cc:126] Checking tablet 8b1b5e519c314a83bf1e39e011440fa6
I20240502 14:15:31.119686 28530 log_verifier.cc:177] Verified matching terms for 302 ops in tablet 8b1b5e519c314a83bf1e39e011440fa6
I20240502 14:15:31.120520 28530 log_verifier.cc:126] Checking tablet 975ae0ae782a45aebda66d2e32713def
I20240502 14:15:31.495198 28530 log_verifier.cc:177] Verified matching terms for 383 ops in tablet 975ae0ae782a45aebda66d2e32713def
I20240502 14:15:31.496058 28530 log_verifier.cc:126] Checking tablet af7e809edf3d44739814918dc2bff263
I20240502 14:15:31.769716 28530 log_verifier.cc:177] Verified matching terms for 244 ops in tablet af7e809edf3d44739814918dc2bff263
I20240502 14:15:31.770363 28530 log_verifier.cc:126] Checking tablet b63847bdc0e546bca632a85255a45572
I20240502 14:15:31.826018 28530 log_verifier.cc:177] Verified matching terms for 4 ops in tablet b63847bdc0e546bca632a85255a45572
I20240502 14:15:31.826125 28530 log_verifier.cc:126] Checking tablet bfa09c15ce944ce29fce3bbc1ad4df56
I20240502 14:15:32.134400 28530 log_verifier.cc:177] Verified matching terms for 277 ops in tablet bfa09c15ce944ce29fce3bbc1ad4df56
I20240502 14:15:32.135123 28530 log_verifier.cc:126] Checking tablet d4aa1eb08d4b450f9c5be499d8c6bbc7
I20240502 14:15:32.444656 28530 log_verifier.cc:177] Verified matching terms for 239 ops in tablet d4aa1eb08d4b450f9c5be499d8c6bbc7
I20240502 14:15:32.448422 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.136:0
--local_ip_for_outbound_sockets=127.27.220.136
--webserver_interface=127.27.220.136
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--builtin_ntp_servers=127.27.220.148:41401
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:15:32.828781 1417 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:32.829067 1417 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:32.829183 1417 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:32.837219 1417 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:32.837373 1417 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:32.837591 1417 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.136
I20240502 14:15:32.847347 1417 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:41401
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.136:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.27.220.136
--webserver_port=0
--tserver_master_addrs=127.27.220.190:33157
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.136
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:32.848661 1417 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:32.850585 1417 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:32.863656 1422 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:32.864176 1423 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:32.864950 1425 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:32.865765 1417 server_base.cc:1034] running on GCE node
I20240502 14:15:32.866338 1417 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:32.867276 1417 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:32.868472 1417 hybrid_clock.cc:648] HybridClock initialized: now 1714659332868429 us; error 41 us; skew 500 ppm
I20240502 14:15:32.868817 1417 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:32.874812 1417 webserver.cc:460] Webserver started at http://127.27.220.136:41971/ using document root <none> and password file <none>
I20240502 14:15:32.875579 1417 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:32.875672 1417 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:32.875996 1417 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:32.878667 1417 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "0f4f2e4ef00d4a648f1ccb69de68771a"
format_stamp: "Formatted at 2024-05-02 14:15:32 on dist-test-slave-j72s"
I20240502 14:15:32.879412 1417 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "0f4f2e4ef00d4a648f1ccb69de68771a"
format_stamp: "Formatted at 2024-05-02 14:15:32 on dist-test-slave-j72s"
I20240502 14:15:32.884613 1417 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.003s
I20240502 14:15:32.888056 1431 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:32.889452 1417 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:32.889714 1417 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "0f4f2e4ef00d4a648f1ccb69de68771a"
format_stamp: "Formatted at 2024-05-02 14:15:32 on dist-test-slave-j72s"
I20240502 14:15:32.889906 1417 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:32.915884 1417 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:32.916838 1417 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:32.917290 1417 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:32.918430 1417 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:32.920400 1417 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:32.920521 1417 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:32.920635 1417 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:32.920764 1417 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
I20240502 14:15:32.960933 1417 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.136:38139
I20240502 14:15:32.960999 1545 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.136:38139 every 8 connection(s)
I20240502 14:15:32.962217 1417 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 14:15:32.962736 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 1417
I20240502 14:15:32.962935 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 14:15:32.979982 1546 heartbeater.cc:347] Connected to a master server at 127.27.220.190:33157
I20240502 14:15:32.980378 1546 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:32.981498 1546 heartbeater.cc:511] Master 127.27.220.190:33157 requested a full tablet report, sending...
I20240502 14:15:32.983289 30426 ts_manager.cc:194] Registered new tserver with Master: 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139)
I20240502 14:15:32.984607 30426 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.136:50105
I20240502 14:15:33.547729 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:33.568696 1481 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:33.582293 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:33.585767 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:33.603713 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:15:33.614689 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:33.637324 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:33.651430 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:33.840961 31204 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:15:33.858951 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:33.877153 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:33.885877 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:33.908944 1481 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:33.929512 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:33.931190 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:33.967378 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:33.987841 1546 heartbeater.cc:503] Master 127.27.220.190:33157 was elected leader, sending a full tablet report...
I20240502 14:15:34.044515 31222 consensus_queue.cc:237] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 302, Committed index: 302, Last appended: 1.302, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:34.047519 30700 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 302. Preceding OpId from leader: term: 1 index: 303. (index mismatch)
I20240502 14:15:34.047857 32156 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 302. Preceding OpId from leader: term: 1 index: 303. (index mismatch)
I20240502 14:15:34.048027 1414 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 303, Last known committed idx: 302, Time since last communication: 0.000s
I20240502 14:15:34.049726 1414 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 303, Last known committed idx: 302, Time since last communication: 0.000s
I20240502 14:15:34.052587 1351 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.303: config changed from index 101 to 303, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.053189 32156 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.303: config changed from index 101 to 303, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.060492 30427 catalog_manager.cc:5484] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 reported cstate change: config changed from index 101 to 303, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.066072 31094 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 302. Preceding OpId from leader: term: 1 index: 303. (index mismatch)
I20240502 14:15:34.066632 30700 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.303: config changed from index 101 to 303, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.070135 1414 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 303, Last known committed idx: 302, Time since last communication: 0.000s
I20240502 14:15:34.073680 31094 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.303: config changed from index 101 to 303, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.084877 30570 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 302. Preceding OpId from leader: term: 1 index: 303. (index mismatch)
I20240502 14:15:34.087026 1414 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 303, Last known committed idx: 302, Time since last communication: 0.000s
I20240502 14:15:34.089286 30570 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.303: config changed from index 101 to 303, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
W20240502 14:15:34.100106 31159 consensus_peers.cc:485] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8b1b5e519c314a83bf1e39e011440fa6. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:34.120440 30570 consensus_queue.cc:237] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 244, Committed index: 244, Last appended: 1.244, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:34.123589 30829 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 244. Preceding OpId from leader: term: 1 index: 245. (index mismatch)
I20240502 14:15:34.123634 31094 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 244. Preceding OpId from leader: term: 1 index: 245. (index mismatch)
I20240502 14:15:34.124186 1432 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 245, Last known committed idx: 244, Time since last communication: 0.000s
I20240502 14:15:34.124568 1659 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 245, Last known committed idx: 244, Time since last communication: 0.000s
I20240502 14:15:34.127758 1659 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.245: config changed from index -1 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.131278 31222 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 244. Preceding OpId from leader: term: 1 index: 245. (index mismatch)
I20240502 14:15:34.132002 30700 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 244. Preceding OpId from leader: term: 1 index: 245. (index mismatch)
I20240502 14:15:34.132184 1432 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 245, Last known committed idx: 244, Time since last communication: 0.000s
I20240502 14:15:34.132509 30829 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.245: config changed from index -1 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.135181 31094 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.245: config changed from index -1 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.139094 1666 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 245, Last known committed idx: 244, Time since last communication: 0.000s
I20240502 14:15:34.139946 30426 catalog_manager.cc:5484] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index -1 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.149389 31222 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.245: config changed from index -1 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
W20240502 14:15:34.153223 30509 consensus_peers.cc:485] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: af7e809edf3d44739814918dc2bff263. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:34.163879 31094 consensus_queue.cc:237] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 244, Committed index: 244, Last appended: 1.244, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:34.166790 30698 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 244. Preceding OpId from leader: term: 1 index: 245. (index mismatch)
I20240502 14:15:34.167455 1416 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 245, Last known committed idx: 244, Time since last communication: 0.000s
I20240502 14:15:34.167341 31222 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 244. Preceding OpId from leader: term: 1 index: 245. (index mismatch)
I20240502 14:15:34.168462 1416 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 245, Last known committed idx: 244, Time since last communication: 0.000s
I20240502 14:15:34.177376 30700 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.245: config changed from index -1 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.182533 30570 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 244. Preceding OpId from leader: term: 1 index: 245. (index mismatch)
I20240502 14:15:34.190832 1676 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 245, Last known committed idx: 244, Time since last communication: 0.000s
I20240502 14:15:34.192999 1676 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.245: config changed from index 78 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.193694 31223 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.245: config changed from index 78 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.199486 30426 catalog_manager.cc:5484] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: config changed from index 78 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.209848 30829 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 244. Preceding OpId from leader: term: 1 index: 245. (index mismatch)
I20240502 14:15:34.210012 30570 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.245: config changed from index 78 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.214372 1677 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 245, Last known committed idx: 244, Time since last communication: 0.001s
I20240502 14:15:34.220443 30570 consensus_queue.cc:237] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:34.221577 30829 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.245: config changed from index 78 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.226907 32156 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 14:15:34.227440 30509 consensus_peers.cc:485] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b63847bdc0e546bca632a85255a45572. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:34.227744 1666 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:15:34.228690 31223 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:15:34.229144 30959 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:15:34.229431 1666 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:15:34.229861 1659 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 14:15:34.230157 31029 consensus_peers.cc:485] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3c26d346223a498bb96b962fa7dd99aa. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:34.231493 31094 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:15:34.232131 1666 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:15:34.244568 1688 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.247625 31223 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.248301 32156 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.248384 30959 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.253595 30431 catalog_manager.cc:5484] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: config changed from index 4 to 5, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.257278 31094 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.284072 31223 consensus_queue.cc:237] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 185, Committed index: 185, Last appended: 1.185, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:34.286912 30700 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 185. Preceding OpId from leader: term: 1 index: 186. (index mismatch)
I20240502 14:15:34.287274 30570 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 185. Preceding OpId from leader: term: 1 index: 186. (index mismatch)
W20240502 14:15:34.287513 31159 consensus_peers.cc:485] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 23cf772ace1c44ecbd450e0bce820c3f. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:34.287772 31094 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 185. Preceding OpId from leader: term: 1 index: 186. (index mismatch)
I20240502 14:15:34.287840 1649 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 186, Last known committed idx: 185, Time since last communication: 0.000s
I20240502 14:15:34.288167 1640 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 186, Last known committed idx: 185, Time since last communication: 0.000s
I20240502 14:15:34.288548 32156 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 185. Preceding OpId from leader: term: 1 index: 186. (index mismatch)
I20240502 14:15:34.288861 1640 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 186, Last known committed idx: 185, Time since last communication: 0.000s
I20240502 14:15:34.291992 1640 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 186, Last known committed idx: 185, Time since last communication: 0.000s
I20240502 14:15:34.304684 1351 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.186: config changed from index 88 to 186, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.306975 31094 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.186: config changed from index 88 to 186, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.308707 32156 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.186: config changed from index 88 to 186, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.309942 30700 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.186: config changed from index 88 to 186, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.314811 30426 catalog_manager.cc:5484] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 reported cstate change: config changed from index 88 to 186, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.323580 30570 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.186: config changed from index 88 to 186, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.672401 1753 ts_tablet_manager.cc:902] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:15:34.672832 30700 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.245: config changed from index 78 to 245, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:34.676599 1753 tablet_copy_client.cc:308] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:15:34.682392 31243 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet 8b1b5e519c314a83bf1e39e011440fa6 from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:51905)
I20240502 14:15:34.682511 31243 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet 8b1b5e519c314a83bf1e39e011440fa6 from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:51905: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-8b1b5e519c314a83bf1e39e011440fa6
I20240502 14:15:34.684687 31243 tablet_copy_source_session.cc:205] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:34.689028 1753 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b1b5e519c314a83bf1e39e011440fa6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:34.698591 1753 tablet_copy_client.cc:783] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:34.698915 1753 tablet_copy_client.cc:647] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:34.701767 1753 tablet_copy_client.cc:523] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:34.704979 1759 ts_tablet_manager.cc:902] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:15:34.705404 1753 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:34.713475 1759 tablet_copy_client.cc:308] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:33055
I20240502 14:15:34.718030 1764 ts_tablet_manager.cc:902] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:15:34.720314 30594 tablet_copy_service.cc:140] P fd3399b1bfe644159a833ede6726c25f: Received BeginTabletCopySession request for tablet af7e809edf3d44739814918dc2bff263 from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:57509)
I20240502 14:15:34.720471 30594 tablet_copy_service.cc:161] P fd3399b1bfe644159a833ede6726c25f: Beginning new tablet copy session on tablet af7e809edf3d44739814918dc2bff263 from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:57509: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-af7e809edf3d44739814918dc2bff263
I20240502 14:15:34.723577 30594 tablet_copy_source_session.cc:205] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:34.725595 1759 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet af7e809edf3d44739814918dc2bff263. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:34.734478 1759 tablet_copy_client.cc:783] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:34.734795 1759 tablet_copy_client.cc:647] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:34.737213 1764 tablet_copy_client.cc:308] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:34805
I20240502 14:15:34.737691 1759 tablet_copy_client.cc:523] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:34.740775 1759 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:34.744678 31114 tablet_copy_service.cc:140] P ba420f2442d6416bbda95edfff85e849: Received BeginTabletCopySession request for tablet 3c26d346223a498bb96b962fa7dd99aa from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:37239)
I20240502 14:15:34.744827 31114 tablet_copy_service.cc:161] P ba420f2442d6416bbda95edfff85e849: Beginning new tablet copy session on tablet 3c26d346223a498bb96b962fa7dd99aa from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:37239: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-3c26d346223a498bb96b962fa7dd99aa
I20240502 14:15:34.747596 31114 tablet_copy_source_session.cc:205] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:34.749140 1764 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3c26d346223a498bb96b962fa7dd99aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:34.755796 1764 tablet_copy_client.cc:783] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:34.756100 1764 tablet_copy_client.cc:647] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:34.759030 1764 tablet_copy_client.cc:523] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:34.762818 1764 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:34.770740 1753 log.cc:824] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: Log is configured to *not* fsync() on all Append() calls
I20240502 14:15:34.812512 1771 ts_tablet_manager.cc:902] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055)
I20240502 14:15:34.824559 1771 tablet_copy_client.cc:308] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:33055
I20240502 14:15:34.825327 30594 tablet_copy_service.cc:140] P fd3399b1bfe644159a833ede6726c25f: Received BeginTabletCopySession request for tablet b63847bdc0e546bca632a85255a45572 from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:57509)
I20240502 14:15:34.825479 30594 tablet_copy_service.cc:161] P fd3399b1bfe644159a833ede6726c25f: Beginning new tablet copy session on tablet b63847bdc0e546bca632a85255a45572 from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:57509: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-b63847bdc0e546bca632a85255a45572
I20240502 14:15:34.828496 30594 tablet_copy_source_session.cc:205] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:34.829733 1771 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b63847bdc0e546bca632a85255a45572. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:34.842401 1777 ts_tablet_manager.cc:902] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:15:34.845221 1771 tablet_copy_client.cc:783] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:34.845479 1771 tablet_copy_client.cc:647] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:34.853235 1771 tablet_copy_client.cc:523] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:34.862848 1771 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:34.881500 1777 tablet_copy_client.cc:308] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:15:34.882994 31243 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet 23cf772ace1c44ecbd450e0bce820c3f from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:51905)
I20240502 14:15:34.883170 31243 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet 23cf772ace1c44ecbd450e0bce820c3f from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:51905: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-23cf772ace1c44ecbd450e0bce820c3f
I20240502 14:15:34.886018 31243 tablet_copy_source_session.cc:205] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:34.897454 1777 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 23cf772ace1c44ecbd450e0bce820c3f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:34.922469 1771 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:34.923020 1771 tablet_bootstrap.cc:492] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:34.923524 1771 ts_tablet_manager.cc:1367] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.061s user 0.004s sys 0.010s
I20240502 14:15:34.937247 1777 tablet_copy_client.cc:783] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:34.937610 1777 tablet_copy_client.cc:647] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:34.938798 1771 raft_consensus.cc:348] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:34.939356 1771 raft_consensus.cc:729] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:34.940412 1777 tablet_copy_client.cc:523] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:34.944001 1777 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:34.953606 1771 consensus_queue.cc:260] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:34.970242 1771 ts_tablet_manager.cc:1398] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.047s user 0.015s sys 0.000s
I20240502 14:15:34.971654 30594 tablet_copy_service.cc:342] P fd3399b1bfe644159a833ede6726c25f: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-b63847bdc0e546bca632a85255a45572 received from {username='slave'} at 127.27.220.136:57509
I20240502 14:15:34.973809 30594 tablet_copy_service.cc:433] P fd3399b1bfe644159a833ede6726c25f: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-b63847bdc0e546bca632a85255a45572 on tablet b63847bdc0e546bca632a85255a45572 with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:35.224854 1501 raft_consensus.cc:1203] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:15:35.238627 1753 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=303 overwritten=0 applied=303 ignored=0} inserts{seen=972 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:35.239202 1753 tablet_bootstrap.cc:492] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:35.239615 1753 ts_tablet_manager.cc:1367] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.534s user 0.379s sys 0.035s
I20240502 14:15:35.240926 1753 raft_consensus.cc:348] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:35.241303 1753 raft_consensus.cc:729] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:35.241586 1753 consensus_queue.cc:260] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 303, Last appended: 1.303, Last appended by leader: 303, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:35.242269 1753 ts_tablet_manager.cc:1398] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:35.242983 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-8b1b5e519c314a83bf1e39e011440fa6 received from {username='slave'} at 127.27.220.136:51905
I20240502 14:15:35.243108 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-8b1b5e519c314a83bf1e39e011440fa6 on tablet 8b1b5e519c314a83bf1e39e011440fa6 with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:35.271677 1764 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=245 overwritten=0 applied=245 ignored=0} inserts{seen=914 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:35.272238 1764 tablet_bootstrap.cc:492] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:35.272568 1764 ts_tablet_manager.cc:1367] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.510s user 0.344s sys 0.035s
I20240502 14:15:35.273363 1764 raft_consensus.cc:348] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:35.273715 1764 raft_consensus.cc:729] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:35.274000 1764 consensus_queue.cc:260] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 245, Last appended: 1.245, Last appended by leader: 245, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:35.274869 1764 ts_tablet_manager.cc:1398] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.002s user 0.005s sys 0.000s
I20240502 14:15:35.275574 31114 tablet_copy_service.cc:342] P ba420f2442d6416bbda95edfff85e849: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-3c26d346223a498bb96b962fa7dd99aa received from {username='slave'} at 127.27.220.136:37239
I20240502 14:15:35.275676 31114 tablet_copy_service.cc:433] P ba420f2442d6416bbda95edfff85e849: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-3c26d346223a498bb96b962fa7dd99aa on tablet 3c26d346223a498bb96b962fa7dd99aa with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:35.316991 1759 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=245 overwritten=0 applied=245 ignored=0} inserts{seen=917 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:35.317564 1759 tablet_bootstrap.cc:492] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:35.317929 1759 ts_tablet_manager.cc:1367] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.577s user 0.281s sys 0.060s
I20240502 14:15:35.319129 1759 raft_consensus.cc:348] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:35.319435 1759 raft_consensus.cc:729] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:35.319689 1759 consensus_queue.cc:260] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 245, Last appended: 1.245, Last appended by leader: 245, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 245 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:35.321931 1759 ts_tablet_manager.cc:1398] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:15:35.322654 30594 tablet_copy_service.cc:342] P fd3399b1bfe644159a833ede6726c25f: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-af7e809edf3d44739814918dc2bff263 received from {username='slave'} at 127.27.220.136:57509
I20240502 14:15:35.322782 30594 tablet_copy_service.cc:433] P fd3399b1bfe644159a833ede6726c25f: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-af7e809edf3d44739814918dc2bff263 on tablet af7e809edf3d44739814918dc2bff263 with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:35.417423 1777 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=186 overwritten=0 applied=186 ignored=0} inserts{seen=868 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:35.418278 1777 tablet_bootstrap.cc:492] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:35.418689 1777 ts_tablet_manager.cc:1367] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.475s user 0.231s sys 0.026s
I20240502 14:15:35.419709 1777 raft_consensus.cc:348] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:35.420120 1777 raft_consensus.cc:729] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:35.420389 1777 consensus_queue.cc:260] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 186, Last appended: 1.186, Last appended by leader: 186, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 186 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:35.422425 1777 ts_tablet_manager.cc:1398] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:15:35.424316 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-23cf772ace1c44ecbd450e0bce820c3f received from {username='slave'} at 127.27.220.136:51905
I20240502 14:15:35.424466 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-23cf772ace1c44ecbd450e0bce820c3f on tablet 23cf772ace1c44ecbd450e0bce820c3f with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:35.530335 1501 raft_consensus.cc:1203] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.302->[1.303-1.303] Dedup: 1.303->[]
I20240502 14:15:35.648896 1501 raft_consensus.cc:1203] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.244->[1.245-1.245] Dedup: 1.245->[]
I20240502 14:15:35.671499 1501 raft_consensus.cc:1203] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.185->[1.186-1.186] Dedup: 1.186->[]
I20240502 14:15:35.785370 1659 raft_consensus.cc:1050] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:35.786228 1659 consensus_queue.cc:237] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:35.788705 32156 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:15:35.789242 1501 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:15:35.789340 1688 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:15:35.789549 31223 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:15:35.790284 1688 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:15:35.790745 1688 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:15:35.791205 1501 raft_consensus.cc:1203] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.244->[1.245-1.245] Dedup: 1.245->[]
I20240502 14:15:35.792740 30959 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:15:35.792824 31094 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:15:35.793498 1688 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:15:35.793784 1659 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:15:35.800418 1688 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.801267 1501 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.805673 31223 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.806090 31094 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.807047 30430 catalog_manager.cc:5484] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a reported cstate change: config changed from index 5 to 6, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.811043 32156 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.813288 30959 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.831187 30570 consensus_queue.cc:237] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:35.833225 31094 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:15:35.833216 31223 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:15:35.833348 1501 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:15:35.833739 32156 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:15:35.833884 1666 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:15:35.834097 1688 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:15:35.834371 1659 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:15:35.834792 1864 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:15:35.836905 1688 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.837579 32156 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.838133 31094 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.840513 30419 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b63847bdc0e546bca632a85255a45572 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:35.840974 31223 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.843060 30433 catalog_manager.cc:5484] T b63847bdc0e546bca632a85255a45572 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 6 to 7, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:35.845937 1501 raft_consensus.cc:2932] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:35.848522 30943 tablet_service.cc:1514] Processing DeleteTablet for tablet b63847bdc0e546bca632a85255a45572 with delete_type TABLET_DATA_TOMBSTONED (TS 7c1f568f8efc4b2a9dbf38907f02f3e3 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:53120
I20240502 14:15:35.850875 1877 tablet_replica.cc:330] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: stopping tablet replica
I20240502 14:15:35.851090 1877 raft_consensus.cc:2229] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:35.851819 1877 raft_consensus.cc:2258] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:35.853539 1877 ts_tablet_manager.cc:1875] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:35.857736 1877 ts_tablet_manager.cc:1888] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:15:35.857821 1877 log.cc:1196] T b63847bdc0e546bca632a85255a45572 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/b63847bdc0e546bca632a85255a45572
I20240502 14:15:35.858342 30418 catalog_manager.cc:4830] TS 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): tablet b63847bdc0e546bca632a85255a45572 (table TestTable [id=9f6d350f0a5941b88f4dc33c6d359c08]) successfully deleted
I20240502 14:15:36.036062 31223 consensus_queue.cc:237] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 277, Committed index: 277, Last appended: 1.277, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:36.043581 30570 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 277. Preceding OpId from leader: term: 1 index: 278. (index mismatch)
I20240502 14:15:36.043581 30959 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 277. Preceding OpId from leader: term: 1 index: 278. (index mismatch)
I20240502 14:15:36.044193 32156 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 277. Preceding OpId from leader: term: 1 index: 278. (index mismatch)
W20240502 14:15:36.044446 31159 consensus_peers.cc:485] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: bfa09c15ce944ce29fce3bbc1ad4df56. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:36.044514 1351 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 278, Last known committed idx: 277, Time since last communication: 0.000s
I20240502 14:15:36.045753 1836 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 278, Last known committed idx: 277, Time since last communication: 0.000s
I20240502 14:15:36.050963 30829 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 277. Preceding OpId from leader: term: 1 index: 278. (index mismatch)
I20240502 14:15:36.052592 1351 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 278, Last known committed idx: 277, Time since last communication: 0.000s
I20240502 14:15:36.056293 1836 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 278, Last known committed idx: 277, Time since last communication: 0.000s
I20240502 14:15:36.057356 1836 raft_consensus.cc:1050] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:36.058624 1836 consensus_queue.cc:237] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 303, Committed index: 303, Last appended: 1.303, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 304 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:36.062091 1351 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.278: config changed from index 118 to 278, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.062808 30829 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.278: config changed from index 118 to 278, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.066946 30959 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.278: config changed from index 118 to 278, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.081653 30431 catalog_manager.cc:5484] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 118 to 278, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.086344 30698 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 303. Preceding OpId from leader: term: 1 index: 304. (index mismatch)
I20240502 14:15:36.087163 1649 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 304, Last known committed idx: 303, Time since last communication: 0.000s
I20240502 14:15:36.098026 1501 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 303. Preceding OpId from leader: term: 1 index: 304. (index mismatch)
I20240502 14:15:36.099267 1649 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 304, Last known committed idx: 303, Time since last communication: 0.000s
I20240502 14:15:36.107439 30570 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 303. Preceding OpId from leader: term: 1 index: 304. (index mismatch)
I20240502 14:15:36.108728 1649 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 304, Last known committed idx: 303, Time since last communication: 0.000s
I20240502 14:15:36.109143 31094 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 303. Preceding OpId from leader: term: 1 index: 304. (index mismatch)
I20240502 14:15:36.113186 32153 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 303. Preceding OpId from leader: term: 1 index: 304. (index mismatch)
I20240502 14:15:36.117843 1649 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 304, Last known committed idx: 303, Time since last communication: 0.004s
I20240502 14:15:36.129106 1649 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 304, Last known committed idx: 303, Time since last communication: 0.000s
I20240502 14:15:36.193962 1666 raft_consensus.cc:1050] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:36.195395 1666 consensus_queue.cc:237] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 245, Committed index: 245, Last appended: 1.245, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:36.200551 1500 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.201241 1688 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.204519 31094 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.205188 1688 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.209883 31222 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.209944 30697 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.210520 30831 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.210588 1666 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.211438 1659 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.218375 1666 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.252801 1649 raft_consensus.cc:1050] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:36.254289 1649 consensus_queue.cc:237] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 186, Committed index: 186, Last appended: 1.186, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 187 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:36.275835 1499 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 186. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
I20240502 14:15:36.277443 30570 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 186. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
I20240502 14:15:36.278308 1923 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 187, Last known committed idx: 186, Time since last communication: 0.000s
I20240502 14:15:36.281741 1923 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 187, Last known committed idx: 186, Time since last communication: 0.000s
I20240502 14:15:36.300230 1501 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.304: config changed from index 303 to 304, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 304 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.304924 1688 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.306227 30697 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.310915 30831 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.316269 30426 catalog_manager.cc:5484] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:15:36.324754 31090 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.348904 31090 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 186. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
I20240502 14:15:36.353749 30570 consensus_queue.cc:237] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 246, Committed index: 246, Last appended: 1.246, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:36.358896 1499 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240502 14:15:36.360316 1659 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.362114 1499 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.366401 31090 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240502 14:15:36.367498 1688 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240502 14:15:36.379868 1666 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [term 1 LEADER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.385341 1498 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.392846 30419 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet af7e809edf3d44739814918dc2bff263 with cas_config_opid_index 246: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:36.394441 30700 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 186. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
I20240502 14:15:36.394232 31090 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.394193 1836 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.304: config changed from index 303 to 304, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 304 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.396852 30431 catalog_manager.cc:5484] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 246 to 247, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "fd3399b1bfe644159a833ede6726c25f" committed_config { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:36.398176 32153 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 186. Preceding OpId from leader: term: 1 index: 187. (index mismatch)
W20240502 14:15:36.398941 31158 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:41511, user_credentials={real_user=slave}} blocked reactor thread for 48936.6us
W20240502 14:15:36.399488 31159 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.129:33055, user_credentials={real_user=slave}} blocked reactor thread for 49076.7us
I20240502 14:15:36.399662 1918 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 187, Last known committed idx: 186, Time since last communication: 0.000s
I20240502 14:15:36.403973 1923 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 187, Last known committed idx: 186, Time since last communication: 0.000s
I20240502 14:15:36.413502 30813 tablet_service.cc:1514] Processing DeleteTablet for tablet af7e809edf3d44739814918dc2bff263 with delete_type TABLET_DATA_TOMBSTONED (TS 155f26c73dc84441a58417c09f577494 not found in new config with opid_index 247) from {username='slave'} at 127.0.0.1:33764
I20240502 14:15:36.420893 1920 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 187, Last known committed idx: 186, Time since last communication: 0.000s
I20240502 14:15:36.422623 31222 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.423341 30699 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.304: config changed from index 303 to 304, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 304 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.426472 32156 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.304: config changed from index 303 to 304, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 304 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.431067 32157 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.278: config changed from index 118 to 278, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.431308 31094 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.304: config changed from index 303 to 304, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 304 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.441390 30426 catalog_manager.cc:5484] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: config changed from index 303 to 304, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 304 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:36.444401 31223 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240502 14:15:36.447791 1945 tablet_replica.cc:330] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: stopping tablet replica
I20240502 14:15:36.448107 1666 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240502 14:15:36.448221 1945 raft_consensus.cc:2229] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:36.448671 1945 raft_consensus.cc:2258] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:36.450448 1918 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.187: config changed from index 186 to 187, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 187 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.452978 1945 ts_tablet_manager.cc:1875] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
W20240502 14:15:36.455998 31159 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.129:33055, user_credentials={real_user=slave}} blocked reactor thread for 52946.8us
I20240502 14:15:36.456794 30570 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.278: config changed from index 118 to 278, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.457388 31090 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.187: config changed from index 186 to 187, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 187 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.464073 32157 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.187: config changed from index 186 to 187, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 187 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.465807 30570 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.304: config changed from index 303 to 304, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 304 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.466377 30426 catalog_manager.cc:5484] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 186 to 187, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 187 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.473105 1952 ts_tablet_manager.cc:902] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:15:36.476912 31222 consensus_queue.cc:237] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 304, Committed index: 304, Last appended: 1.304, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 305 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:36.476723 31223 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.476933 30704 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.187: config changed from index 186 to 187, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 187 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.479923 31090 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 304. Preceding OpId from leader: term: 1 index: 305. (index mismatch)
I20240502 14:15:36.480626 1943 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 305, Last known committed idx: 304, Time since last communication: 0.000s
I20240502 14:15:36.483167 30572 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 304. Preceding OpId from leader: term: 1 index: 305. (index mismatch)
I20240502 14:15:36.484422 1499 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 304. Preceding OpId from leader: term: 1 index: 305. (index mismatch)
I20240502 14:15:36.485200 1501 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.187: config changed from index 186 to 187, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 187 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.488294 30698 raft_consensus.cc:1261] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer fd3399b1bfe644159a833ede6726c25f: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240502 14:15:36.489727 1666 consensus_queue.cc:1035] T af7e809edf3d44739814918dc2bff263 P fd3399b1bfe644159a833ede6726c25f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240502 14:15:36.491168 30570 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.187: config changed from index 186 to 187, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 187 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false replace: true } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.491768 1416 raft_consensus.cc:1050] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:36.493209 1416 consensus_queue.cc:237] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 245, Committed index: 245, Last appended: 1.245, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:36.496838 32157 raft_consensus.cc:1261] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 304. Preceding OpId from leader: term: 1 index: 305. (index mismatch)
I20240502 14:15:36.496865 1943 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 305, Last known committed idx: 304, Time since last communication: 0.000s
I20240502 14:15:36.499792 30697 raft_consensus.cc:2932] T af7e809edf3d44739814918dc2bff263 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.500211 1943 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 305, Last known committed idx: 304, Time since last communication: 0.000s
I20240502 14:15:36.501376 1649 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.305: config changed from index 304 to 305, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 305 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.509665 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8b1b5e519c314a83bf1e39e011440fa6 with cas_config_opid_index 304: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:36.509958 1836 consensus_queue.cc:1035] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 305, Last known committed idx: 304, Time since last communication: 0.000s
I20240502 14:15:36.512888 30572 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.305: config changed from index 304 to 305, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 305 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.513700 32157 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.305: config changed from index 304 to 305, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 305 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.514057 31094 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.305: config changed from index 304 to 305, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 305 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.520015 30427 catalog_manager.cc:5484] T 8b1b5e519c314a83bf1e39e011440fa6 P 3eafb09239024de3bc0eea8d700f7668 reported cstate change: config changed from index 304 to 305, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 305 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.523320 1499 raft_consensus.cc:2932] T 8b1b5e519c314a83bf1e39e011440fa6 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.305: config changed from index 304 to 305, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 305 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.525024 30682 tablet_service.cc:1514] Processing DeleteTablet for tablet 8b1b5e519c314a83bf1e39e011440fa6 with delete_type TABLET_DATA_TOMBSTONED (TS d733c51902044956a8cd2b6b31bc9831 not found in new config with opid_index 305) from {username='slave'} at 127.0.0.1:39850
I20240502 14:15:36.527662 31223 consensus_queue.cc:237] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 187, Committed index: 187, Last appended: 1.187, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:36.530929 31090 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 14:15:36.531597 1918 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 14:15:36.532836 1500 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 14:15:36.536113 30572 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 14:15:36.537084 30698 raft_consensus.cc:1261] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 14:15:36.538154 1918 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 14:15:36.542279 1918 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 14:15:36.542832 1918 consensus_queue.cc:1035] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 14:15:36.547387 1952 tablet_copy_client.cc:308] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:15:36.548190 31243 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet bfa09c15ce944ce29fce3bbc1ad4df56 from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:51905)
I20240502 14:15:36.548369 31243 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet bfa09c15ce944ce29fce3bbc1ad4df56 from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:51905: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-bfa09c15ce944ce29fce3bbc1ad4df56
I20240502 14:15:36.550242 1918 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.188: config changed from index 187 to 188, VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) evicted. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.551147 31243 tablet_copy_source_session.cc:205] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:36.554180 1959 tablet_replica.cc:330] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: stopping tablet replica
I20240502 14:15:36.554394 1959 raft_consensus.cc:2229] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:36.554163 1499 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 187 to 188, VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) evicted. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.555022 1959 raft_consensus.cc:2258] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:36.556002 1952 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet bfa09c15ce944ce29fce3bbc1ad4df56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:36.556139 31090 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 187 to 188, VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) evicted. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.557793 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 23cf772ace1c44ecbd450e0bce820c3f with cas_config_opid_index 187: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:36.558645 30704 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.559113 30698 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 187 to 188, VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) evicted. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.559948 30431 catalog_manager.cc:5484] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: config changed from index 187 to 188, VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:36.561990 1936 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
W20240502 14:15:36.563261 30767 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:46259, user_credentials={real_user=slave}} blocked reactor thread for 59555.4us
I20240502 14:15:36.565541 31223 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.566367 1930 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.571631 1952 tablet_copy_client.cc:783] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:36.573812 30570 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.573945 30831 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.573248 1501 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 245. Preceding OpId from leader: term: 1 index: 246. (index mismatch)
I20240502 14:15:36.571941 1952 tablet_copy_client.cc:647] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:36.574949 1940 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.575594 1940 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.577728 1940 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 246, Last known committed idx: 245, Time since last communication: 0.000s
I20240502 14:15:36.593227 1959 ts_tablet_manager.cc:1875] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:36.599488 1952 tablet_copy_client.cc:523] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:36.603437 1952 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:36.605111 1945 ts_tablet_manager.cc:1888] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.246
I20240502 14:15:36.613386 1945 log.cc:1196] T af7e809edf3d44739814918dc2bff263 P 155f26c73dc84441a58417c09f577494: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/af7e809edf3d44739814918dc2bff263
I20240502 14:15:36.614061 30572 raft_consensus.cc:2932] T 23cf772ace1c44ecbd450e0bce820c3f P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 187 to 188, VOTER 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135) evicted. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.624732 1940 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.627239 31222 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.628937 1959 ts_tablet_manager.cc:1888] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.304
I20240502 14:15:36.629047 1959 log.cc:1196] T 8b1b5e519c314a83bf1e39e011440fa6 P d733c51902044956a8cd2b6b31bc9831: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/8b1b5e519c314a83bf1e39e011440fa6
I20240502 14:15:36.631762 30431 catalog_manager.cc:5484] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:36.640246 1501 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.640795 30829 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.639698 30570 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.655220 30417 catalog_manager.cc:4830] TS 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): tablet af7e809edf3d44739814918dc2bff263 (table rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]) successfully deleted
I20240502 14:15:36.660210 30417 catalog_manager.cc:4830] TS d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): tablet 8b1b5e519c314a83bf1e39e011440fa6 (table rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]) successfully deleted
I20240502 14:15:36.664614 30704 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.246: config changed from index 245 to 246, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 246 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.673226 31094 consensus_queue.cc:237] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 246, Committed index: 246, Last appended: 1.246, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:36.675799 1501 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240502 14:15:36.676508 30704 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240502 14:15:36.676800 30570 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240502 14:15:36.676951 1940 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240502 14:15:36.677489 1940 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240502 14:15:36.677817 1936 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240502 14:15:36.679453 30831 raft_consensus.cc:1261] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer ba420f2442d6416bbda95edfff85e849: Log matching property violated. Preceding OpId in replica: term: 1 index: 246. Preceding OpId from leader: term: 1 index: 247. (index mismatch)
I20240502 14:15:36.679997 1940 consensus_queue.cc:1035] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 247, Last known committed idx: 246, Time since last communication: 0.000s
I20240502 14:15:36.682936 1940 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 [term 1 LEADER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.683830 30572 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.685127 30829 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.692694 30704 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.692021 30431 catalog_manager.cc:5484] T 3c26d346223a498bb96b962fa7dd99aa P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 246 to 247, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New cstate: current_term: 1 leader_uuid: "ba420f2442d6416bbda95edfff85e849" committed_config { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.694962 30418 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3c26d346223a498bb96b962fa7dd99aa with cas_config_opid_index 246: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:36.698549 1501 raft_consensus.cc:2932] T 3c26d346223a498bb96b962fa7dd99aa P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.247: config changed from index 246 to 247, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:36.713187 31204 tablet_service.cc:1514] Processing DeleteTablet for tablet 3c26d346223a498bb96b962fa7dd99aa with delete_type TABLET_DATA_TOMBSTONED (TS 0dfd48d0b03a42f689e58d5190d03a2f not found in new config with opid_index 247) from {username='slave'} at 127.0.0.1:56766
I20240502 14:15:36.716948 32138 tablet_service.cc:1514] Processing DeleteTablet for tablet 23cf772ace1c44ecbd450e0bce820c3f with delete_type TABLET_DATA_TOMBSTONED (TS 3eafb09239024de3bc0eea8d700f7668 not found in new config with opid_index 188) from {username='slave'} at 127.0.0.1:43546
I20240502 14:15:36.730096 1982 tablet_replica.cc:330] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: stopping tablet replica
I20240502 14:15:36.730433 1982 raft_consensus.cc:2229] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:36.730796 1982 raft_consensus.cc:2258] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:36.731757 1983 tablet_replica.cc:330] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: stopping tablet replica
I20240502 14:15:36.732527 1983 raft_consensus.cc:2229] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:36.733036 1983 raft_consensus.cc:2258] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:36.740435 1983 ts_tablet_manager.cc:1875] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:36.743507 1982 ts_tablet_manager.cc:1875] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:36.762673 1983 ts_tablet_manager.cc:1888] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.187
I20240502 14:15:36.762856 1983 log.cc:1196] T 23cf772ace1c44ecbd450e0bce820c3f P 3eafb09239024de3bc0eea8d700f7668: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/wals/23cf772ace1c44ecbd450e0bce820c3f
I20240502 14:15:36.763857 30417 catalog_manager.cc:4830] TS 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091): tablet 23cf772ace1c44ecbd450e0bce820c3f (table rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]) successfully deleted
I20240502 14:15:36.774175 1982 ts_tablet_manager.cc:1888] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.246
I20240502 14:15:36.774389 1982 log.cc:1196] T 3c26d346223a498bb96b962fa7dd99aa P 0dfd48d0b03a42f689e58d5190d03a2f: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/wals/3c26d346223a498bb96b962fa7dd99aa
I20240502 14:15:36.775449 30420 catalog_manager.cc:4830] TS 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403): tablet 3c26d346223a498bb96b962fa7dd99aa (table rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]) successfully deleted
I20240502 14:15:36.872185 30959 consensus_queue.cc:237] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 269, Committed index: 269, Last appended: 1.269, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:36.875478 32157 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 269. Preceding OpId from leader: term: 1 index: 270. (index mismatch)
I20240502 14:15:36.881569 1874 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 270, Last known committed idx: 269, Time since last communication: 0.000s
I20240502 14:15:36.897311 30572 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 269. Preceding OpId from leader: term: 1 index: 270. (index mismatch)
I20240502 14:15:36.897574 31223 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 269. Preceding OpId from leader: term: 1 index: 270. (index mismatch)
I20240502 14:15:36.898465 2004 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 270, Last known committed idx: 269, Time since last communication: 0.000s
I20240502 14:15:36.899135 2004 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 270, Last known committed idx: 269, Time since last communication: 0.000s
I20240502 14:15:36.911751 31090 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 269. Preceding OpId from leader: term: 1 index: 270. (index mismatch)
I20240502 14:15:36.912505 2004 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 270, Last known committed idx: 269, Time since last communication: 0.000s
W20240502 14:15:36.920087 30899 consensus_peers.cc:485] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 50f7b19e515e497c9a27e8e94cb73079. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:36.921574 1874 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.270: config changed from index 129 to 270, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.930953 31222 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.270: config changed from index 129 to 270, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.939110 30431 catalog_manager.cc:5484] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: config changed from index 129 to 270, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.954190 31090 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.270: config changed from index 129 to 270, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.956128 32157 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.270: config changed from index 129 to 270, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.957770 30572 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.270: config changed from index 129 to 270, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:36.996872 30831 consensus_queue.cc:237] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 224, Committed index: 224, Last appended: 1.224, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:37.000454 32157 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 224. Preceding OpId from leader: term: 1 index: 225. (index mismatch)
I20240502 14:15:37.001688 31090 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 224. Preceding OpId from leader: term: 1 index: 225. (index mismatch)
I20240502 14:15:37.002470 1958 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 225, Last known committed idx: 224, Time since last communication: 0.000s
I20240502 14:15:37.003798 1876 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 225, Last known committed idx: 224, Time since last communication: 0.000s
I20240502 14:15:37.015298 30704 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 224. Preceding OpId from leader: term: 1 index: 225. (index mismatch)
I20240502 14:15:37.015869 1958 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 225, Last known committed idx: 224, Time since last communication: 0.000s
I20240502 14:15:37.018253 30959 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 224. Preceding OpId from leader: term: 1 index: 225. (index mismatch)
I20240502 14:15:37.018812 1958 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 225, Last known committed idx: 224, Time since last communication: 0.000s
I20240502 14:15:37.021237 1958 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 LEADER]: Committing config change with OpId 1.225: config changed from index 57 to 225, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
W20240502 14:15:37.025621 30769 consensus_peers.cc:485] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0f906c7d775540e6ba76cf2ff3700a67. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:37.026285 31090 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.225: config changed from index 57 to 225, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.028306 30959 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.225: config changed from index 57 to 225, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.033398 30429 catalog_manager.cc:5484] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 57 to 225, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "155f26c73dc84441a58417c09f577494" committed_config { opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.043676 32157 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.225: config changed from index 57 to 225, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.052146 30704 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.225: config changed from index 57 to 225, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.073760 30959 consensus_queue.cc:237] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 416, Committed index: 416, Last appended: 1.416, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
W20240502 14:15:37.077080 30899 consensus_peers.cc:485] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 74c46ec356414d26987e1cd6a0857602. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:37.077144 30831 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 416. Preceding OpId from leader: term: 1 index: 417. (index mismatch)
I20240502 14:15:37.077287 32157 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 416. Preceding OpId from leader: term: 1 index: 417. (index mismatch)
I20240502 14:15:37.077759 2012 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 417, Last known committed idx: 416, Time since last communication: 0.000s
I20240502 14:15:37.078065 1874 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 417, Last known committed idx: 416, Time since last communication: 0.000s
I20240502 14:15:37.079005 31090 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 416. Preceding OpId from leader: term: 1 index: 417. (index mismatch)
I20240502 14:15:37.079527 30572 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 416. Preceding OpId from leader: term: 1 index: 417. (index mismatch)
I20240502 14:15:37.080065 2011 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 417, Last known committed idx: 416, Time since last communication: 0.000s
I20240502 14:15:37.081118 2011 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 417, Last known committed idx: 416, Time since last communication: 0.000s
I20240502 14:15:37.090965 2012 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.417: config changed from index 215 to 417, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.091712 31090 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.417: config changed from index 215 to 417, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.091828 32157 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.417: config changed from index 215 to 417, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.096632 30831 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.417: config changed from index 215 to 417, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.098436 30429 catalog_manager.cc:5484] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 reported cstate change: config changed from index 215 to 417, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.102361 30572 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.417: config changed from index 215 to 417, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.110343 1952 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=278 overwritten=0 applied=278 ignored=0} inserts{seen=906 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:37.110803 1952 tablet_bootstrap.cc:492] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:37.111148 1952 ts_tablet_manager.cc:1367] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.508s user 0.347s sys 0.038s
I20240502 14:15:37.111812 1952 raft_consensus.cc:348] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:37.112147 1952 raft_consensus.cc:729] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:37.112386 1952 consensus_queue.cc:260] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 278, Last appended: 1.278, Last appended by leader: 278, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 278 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:37.113157 1952 ts_tablet_manager.cc:1398] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.002s user 0.000s sys 0.003s
I20240502 14:15:37.113919 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-bfa09c15ce944ce29fce3bbc1ad4df56 received from {username='slave'} at 127.27.220.136:51905
I20240502 14:15:37.114053 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-bfa09c15ce944ce29fce3bbc1ad4df56 on tablet bfa09c15ce944ce29fce3bbc1ad4df56 with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:37.177937 31223 consensus_queue.cc:237] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 292, Committed index: 292, Last appended: 1.292, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:37.180740 32157 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 292. Preceding OpId from leader: term: 1 index: 293. (index mismatch)
I20240502 14:15:37.180774 30831 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 292. Preceding OpId from leader: term: 1 index: 293. (index mismatch)
W20240502 14:15:37.181094 31159 consensus_peers.cc:485] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139): Couldn't send request to peer 0f4f2e4ef00d4a648f1ccb69de68771a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 356c1606c56147a2868e60ab63c59c13. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:37.181422 1943 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 293, Last known committed idx: 292, Time since last communication: 0.000s
I20240502 14:15:37.181919 30959 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 292. Preceding OpId from leader: term: 1 index: 293. (index mismatch)
I20240502 14:15:37.182458 1836 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 293, Last known committed idx: 292, Time since last communication: 0.000s
I20240502 14:15:37.182945 1836 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 293, Last known committed idx: 292, Time since last communication: 0.000s
I20240502 14:15:37.190809 1943 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.293: config changed from index 109 to 293, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.193099 30831 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.293: config changed from index 109 to 293, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.196172 30704 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 292. Preceding OpId from leader: term: 1 index: 293. (index mismatch)
I20240502 14:15:37.196775 1649 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 293, Last known committed idx: 292, Time since last communication: 0.000s
I20240502 14:15:37.201122 30431 catalog_manager.cc:5484] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 109 to 293, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.204140 30704 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.293: config changed from index 109 to 293, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.209231 30959 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.293: config changed from index 109 to 293, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.218168 32157 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.293: config changed from index 109 to 293, NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } } }
I20240502 14:15:37.391089 1952 ts_tablet_manager.cc:902] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975)
I20240502 14:15:37.394444 1952 tablet_copy_client.cc:308] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:34975
I20240502 14:15:37.400002 30854 tablet_copy_service.cc:140] P 155f26c73dc84441a58417c09f577494: Received BeginTabletCopySession request for tablet 0f906c7d775540e6ba76cf2ff3700a67 from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:49607)
I20240502 14:15:37.400137 30854 tablet_copy_service.cc:161] P 155f26c73dc84441a58417c09f577494: Beginning new tablet copy session on tablet 0f906c7d775540e6ba76cf2ff3700a67 from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:49607: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-0f906c7d775540e6ba76cf2ff3700a67
I20240502 14:15:37.402082 30854 tablet_copy_source_session.cc:205] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:37.403219 1952 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f906c7d775540e6ba76cf2ff3700a67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:37.407668 1952 tablet_copy_client.cc:783] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:37.407922 1952 tablet_copy_client.cc:647] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:37.410058 1952 tablet_copy_client.cc:523] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:37.412474 1952 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:37.417414 2104 ts_tablet_manager.cc:902] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:15:37.419507 2104 tablet_copy_client.cc:308] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:41511
I20240502 14:15:37.426054 30983 tablet_copy_service.cc:140] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Received BeginTabletCopySession request for tablet 50f7b19e515e497c9a27e8e94cb73079 from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:50951)
I20240502 14:15:37.426215 30983 tablet_copy_service.cc:161] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Beginning new tablet copy session on tablet 50f7b19e515e497c9a27e8e94cb73079 from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:50951: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-50f7b19e515e497c9a27e8e94cb73079
I20240502 14:15:37.428319 30983 tablet_copy_source_session.cc:205] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:37.429613 2104 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 50f7b19e515e497c9a27e8e94cb73079. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:37.434386 2104 tablet_copy_client.cc:783] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:37.434645 2104 tablet_copy_client.cc:647] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:37.437253 2104 tablet_copy_client.cc:523] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:37.440140 2104 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:37.581399 2122 ts_tablet_manager.cc:902] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403)
I20240502 14:15:37.585611 1501 raft_consensus.cc:1203] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.277->[1.278-1.278] Dedup: 1.278->[]
I20240502 14:15:37.622412 2122 tablet_copy_client.cc:308] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:39403
I20240502 14:15:37.623167 31243 tablet_copy_service.cc:140] P 0dfd48d0b03a42f689e58d5190d03a2f: Received BeginTabletCopySession request for tablet 356c1606c56147a2868e60ab63c59c13 from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:51905)
I20240502 14:15:37.623335 31243 tablet_copy_service.cc:161] P 0dfd48d0b03a42f689e58d5190d03a2f: Beginning new tablet copy session on tablet 356c1606c56147a2868e60ab63c59c13 from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:51905: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-356c1606c56147a2868e60ab63c59c13
I20240502 14:15:37.625291 31243 tablet_copy_source_session.cc:205] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:37.626622 2122 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 356c1606c56147a2868e60ab63c59c13. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:37.635360 2122 tablet_copy_client.cc:783] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:37.635639 2122 tablet_copy_client.cc:647] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:37.639539 2122 tablet_copy_client.cc:523] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:37.643204 2122 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:37.696233 2134 ts_tablet_manager.cc:902] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: Initiating tablet copy from peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511)
I20240502 14:15:37.734974 2134 tablet_copy_client.cc:308] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:41511
I20240502 14:15:37.735687 30983 tablet_copy_service.cc:140] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Received BeginTabletCopySession request for tablet 74c46ec356414d26987e1cd6a0857602 from peer 0f4f2e4ef00d4a648f1ccb69de68771a ({username='slave'} at 127.27.220.136:50951)
I20240502 14:15:37.735841 30983 tablet_copy_service.cc:161] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Beginning new tablet copy session on tablet 74c46ec356414d26987e1cd6a0857602 from peer 0f4f2e4ef00d4a648f1ccb69de68771a at {username='slave'} at 127.27.220.136:50951: session id = 0f4f2e4ef00d4a648f1ccb69de68771a-74c46ec356414d26987e1cd6a0857602
I20240502 14:15:37.738643 30983 tablet_copy_source_session.cc:205] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:15:37.740847 2134 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 74c46ec356414d26987e1cd6a0857602. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:37.749481 2134 tablet_copy_client.cc:783] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 0 data blocks...
I20240502 14:15:37.749866 2134 tablet_copy_client.cc:647] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Starting download of 1 WAL segments...
I20240502 14:15:37.753979 2134 tablet_copy_client.cc:523] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:15:37.761114 2134 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap starting.
I20240502 14:15:37.769378 1952 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=225 overwritten=0 applied=225 ignored=0} inserts{seen=853 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:37.769910 1952 tablet_bootstrap.cc:492] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:37.770293 1952 ts_tablet_manager.cc:1367] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.358s user 0.293s sys 0.022s
I20240502 14:15:37.771329 1952 raft_consensus.cc:348] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:37.772308 1952 raft_consensus.cc:729] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:37.772568 1952 consensus_queue.cc:260] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 225, Last appended: 1.225, Last appended by leader: 225, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 225 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:37.773360 1952 ts_tablet_manager.cc:1398] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:15:37.774145 30854 tablet_copy_service.cc:342] P 155f26c73dc84441a58417c09f577494: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-0f906c7d775540e6ba76cf2ff3700a67 received from {username='slave'} at 127.27.220.136:49607
I20240502 14:15:37.774291 30854 tablet_copy_service.cc:433] P 155f26c73dc84441a58417c09f577494: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-0f906c7d775540e6ba76cf2ff3700a67 on tablet 0f906c7d775540e6ba76cf2ff3700a67 with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:37.829195 1501 raft_consensus.cc:1203] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.224->[1.225-1.225] Dedup: 1.225->[]
I20240502 14:15:37.885850 2104 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=270 overwritten=0 applied=270 ignored=0} inserts{seen=952 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:37.886353 2104 tablet_bootstrap.cc:492] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:37.886787 2104 ts_tablet_manager.cc:1367] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.447s user 0.332s sys 0.043s
I20240502 14:15:37.887755 2104 raft_consensus.cc:348] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:37.888057 2104 raft_consensus.cc:729] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:37.888329 2104 consensus_queue.cc:260] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 270, Last appended: 1.270, Last appended by leader: 270, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 270 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:37.888993 2104 ts_tablet_manager.cc:1398] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:15:37.891722 30983 tablet_copy_service.cc:342] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-50f7b19e515e497c9a27e8e94cb73079 received from {username='slave'} at 127.27.220.136:50951
I20240502 14:15:37.891875 30983 tablet_copy_service.cc:433] P 7c1f568f8efc4b2a9dbf38907f02f3e3: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-50f7b19e515e497c9a27e8e94cb73079 on tablet 50f7b19e515e497c9a27e8e94cb73079 with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:37.923643 1501 raft_consensus.cc:1203] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.269->[1.270-1.270] Dedup: 1.270->[]
I20240502 14:15:38.093196 2122 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=293 overwritten=0 applied=293 ignored=0} inserts{seen=921 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:38.093787 2122 tablet_bootstrap.cc:492] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:38.094137 2122 ts_tablet_manager.cc:1367] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.451s user 0.328s sys 0.057s
I20240502 14:15:38.095052 2122 raft_consensus.cc:348] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:38.095352 2122 raft_consensus.cc:729] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:38.095579 2122 consensus_queue.cc:260] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 293, Last appended: 1.293, Last appended by leader: 293, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:38.096295 2122 ts_tablet_manager.cc:1398] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:15:38.097533 31243 tablet_copy_service.cc:342] P 0dfd48d0b03a42f689e58d5190d03a2f: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-356c1606c56147a2868e60ab63c59c13 received from {username='slave'} at 127.27.220.136:51905
I20240502 14:15:38.097678 31243 tablet_copy_service.cc:433] P 0dfd48d0b03a42f689e58d5190d03a2f: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-356c1606c56147a2868e60ab63c59c13 on tablet 356c1606c56147a2868e60ab63c59c13 with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:38.101346 1501 raft_consensus.cc:1203] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.292->[1.293-1.293] Dedup: 1.293->[]
I20240502 14:15:38.164861 2069 raft_consensus.cc:1050] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:38.165971 2069 consensus_queue.cc:237] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 278, Committed index: 278, Last appended: 1.278, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 279 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:38.168656 30959 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 278. Preceding OpId from leader: term: 1 index: 279. (index mismatch)
I20240502 14:15:38.169301 30831 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 278. Preceding OpId from leader: term: 1 index: 279. (index mismatch)
I20240502 14:15:38.169332 1943 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 279, Last known committed idx: 278, Time since last communication: 0.000s
I20240502 14:15:38.170250 1943 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 279, Last known committed idx: 278, Time since last communication: 0.000s
I20240502 14:15:38.172883 1501 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 278. Preceding OpId from leader: term: 1 index: 279. (index mismatch)
I20240502 14:15:38.174012 32157 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 278. Preceding OpId from leader: term: 1 index: 279. (index mismatch)
I20240502 14:15:38.174561 1943 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 279, Last known committed idx: 278, Time since last communication: 0.000s
I20240502 14:15:38.175045 1943 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 279, Last known committed idx: 278, Time since last communication: 0.000s
I20240502 14:15:38.184474 30572 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 278. Preceding OpId from leader: term: 1 index: 279. (index mismatch)
I20240502 14:15:38.185014 1943 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 279, Last known committed idx: 278, Time since last communication: 0.000s
I20240502 14:15:38.195914 2069 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.279: config changed from index 278 to 279, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 279 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.197708 30572 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.279: config changed from index 278 to 279, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 279 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.202157 30831 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.279: config changed from index 278 to 279, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 279 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.206600 32157 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.279: config changed from index 278 to 279, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 279 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.208427 30431 catalog_manager.cc:5484] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f reported cstate change: config changed from index 278 to 279, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 279 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.200170 1501 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.279: config changed from index 278 to 279, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 279 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.216425 30959 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.279: config changed from index 278 to 279, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 279 OBSOLETE_local: false peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.226326 31223 consensus_queue.cc:237] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 279, Committed index: 279, Last appended: 1.279, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 280 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:38.228808 1501 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 279. Preceding OpId from leader: term: 1 index: 280. (index mismatch)
I20240502 14:15:38.229403 30959 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 279. Preceding OpId from leader: term: 1 index: 280. (index mismatch)
I20240502 14:15:38.229566 32157 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 279. Preceding OpId from leader: term: 1 index: 280. (index mismatch)
I20240502 14:15:38.229660 30831 raft_consensus.cc:1261] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 279. Preceding OpId from leader: term: 1 index: 280. (index mismatch)
I20240502 14:15:38.229930 2179 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 280, Last known committed idx: 279, Time since last communication: 0.000s
I20240502 14:15:38.230281 1943 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 280, Last known committed idx: 279, Time since last communication: 0.000s
I20240502 14:15:38.230567 2183 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 280, Last known committed idx: 279, Time since last communication: 0.000s
I20240502 14:15:38.230832 2069 consensus_queue.cc:1035] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 280, Last known committed idx: 279, Time since last communication: 0.000s
I20240502 14:15:38.235059 2069 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.280: config changed from index 279 to 280, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 280 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.236116 1499 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.280: config changed from index 279 to 280, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 280 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.238780 30959 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.280: config changed from index 279 to 280, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 280 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.240160 30420 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet bfa09c15ce944ce29fce3bbc1ad4df56 with cas_config_opid_index 279: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:38.241232 30427 catalog_manager.cc:5484] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: config changed from index 279 to 280, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 280 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:38.241376 30831 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.280: config changed from index 279 to 280, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 280 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.246970 32157 raft_consensus.cc:2932] T bfa09c15ce944ce29fce3bbc1ad4df56 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.280: config changed from index 279 to 280, VOTER fd3399b1bfe644159a833ede6726c25f (127.27.220.129) evicted. New config: { opid_index: 280 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.249161 30553 tablet_service.cc:1514] Processing DeleteTablet for tablet bfa09c15ce944ce29fce3bbc1ad4df56 with delete_type TABLET_DATA_TOMBSTONED (TS fd3399b1bfe644159a833ede6726c25f not found in new config with opid_index 280) from {username='slave'} at 127.0.0.1:38096
I20240502 14:15:38.250000 2189 tablet_replica.cc:330] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: stopping tablet replica
I20240502 14:15:38.250285 2189 raft_consensus.cc:2229] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:38.251269 2189 raft_consensus.cc:2258] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:38.254354 2189 ts_tablet_manager.cc:1875] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:38.260525 2189 ts_tablet_manager.cc:1888] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.279
I20240502 14:15:38.260650 2189 log.cc:1196] T bfa09c15ce944ce29fce3bbc1ad4df56 P fd3399b1bfe644159a833ede6726c25f: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/bfa09c15ce944ce29fce3bbc1ad4df56
I20240502 14:15:38.261546 30419 catalog_manager.cc:4830] TS fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): tablet bfa09c15ce944ce29fce3bbc1ad4df56 (table rebalance_test_table_4 [id=524e8e464d784197b8e14eac16bb56bb]) successfully deleted
I20240502 14:15:38.268934 2028 raft_consensus.cc:1050] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:38.270040 2028 consensus_queue.cc:237] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 225, Committed index: 225, Last appended: 1.225, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 226 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:38.272754 1501 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 225. Preceding OpId from leader: term: 1 index: 226. (index mismatch)
I20240502 14:15:38.274420 30698 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 225. Preceding OpId from leader: term: 1 index: 226. (index mismatch)
I20240502 14:15:38.275053 31090 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 225. Preceding OpId from leader: term: 1 index: 226. (index mismatch)
I20240502 14:15:38.275687 32157 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 225. Preceding OpId from leader: term: 1 index: 226. (index mismatch)
I20240502 14:15:38.276151 2028 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 226, Last known committed idx: 225, Time since last communication: 0.001s
I20240502 14:15:38.277774 2021 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 226, Last known committed idx: 225, Time since last communication: 0.000s
I20240502 14:15:38.278318 2021 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 226, Last known committed idx: 225, Time since last communication: 0.000s
I20240502 14:15:38.299800 30959 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 225. Preceding OpId from leader: term: 1 index: 226. (index mismatch)
I20240502 14:15:38.300714 2028 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 226, Last known committed idx: 225, Time since last communication: 0.000s
I20240502 14:15:38.303189 2028 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 226, Last known committed idx: 225, Time since last communication: 0.000s
I20240502 14:15:38.304952 2134 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap replayed 1/1 log segments. Stats: ops{read=417 overwritten=0 applied=417 ignored=0} inserts{seen=1099 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:15:38.305403 2134 tablet_bootstrap.cc:492] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: Bootstrap complete.
I20240502 14:15:38.305833 2134 ts_tablet_manager.cc:1367] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent bootstrapping tablet: real 0.548s user 0.476s sys 0.051s
I20240502 14:15:38.306830 2134 raft_consensus.cc:348] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:38.307273 2134 raft_consensus.cc:729] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Initialized, Role: LEARNER
I20240502 14:15:38.307577 2134 consensus_queue.cc:260] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 417, Last appended: 1.417, Last appended by leader: 417, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 417 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: true } }
I20240502 14:15:38.308387 2134 ts_tablet_manager.cc:1398] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:15:38.309065 2011 raft_consensus.cc:1050] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:38.310295 2011 consensus_queue.cc:237] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 270, Committed index: 270, Last appended: 1.270, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 271 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:38.310417 2021 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 LEADER]: Committing config change with OpId 1.226: config changed from index 225 to 226, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 226 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.313020 30983 tablet_copy_service.cc:342] P 7c1f568f8efc4b2a9dbf38907f02f3e3: Request end of tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-74c46ec356414d26987e1cd6a0857602 received from {username='slave'} at 127.27.220.136:50951
I20240502 14:15:38.313184 30983 tablet_copy_service.cc:433] P 7c1f568f8efc4b2a9dbf38907f02f3e3: ending tablet copy session 0f4f2e4ef00d4a648f1ccb69de68771a-74c46ec356414d26987e1cd6a0857602 on tablet 74c46ec356414d26987e1cd6a0857602 with peer 0f4f2e4ef00d4a648f1ccb69de68771a
I20240502 14:15:38.317373 1499 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.226: config changed from index 225 to 226, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 226 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.324790 31090 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.226: config changed from index 225 to 226, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 226 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.326090 30959 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.226: config changed from index 225 to 226, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 226 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.330168 30429 catalog_manager.cc:5484] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a reported cstate change: config changed from index 225 to 226, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "155f26c73dc84441a58417c09f577494" committed_config { opid_index: 226 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.354563 1501 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 1 index: 271. (index mismatch)
I20240502 14:15:38.355324 2149 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240502 14:15:38.374840 32156 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 1 index: 271. (index mismatch)
I20240502 14:15:38.375902 2149 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240502 14:15:38.383520 30698 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.226: config changed from index 225 to 226, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 226 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.385900 32157 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.226: config changed from index 225 to 226, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 226 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } attrs { replace: true } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.387328 30572 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 1 index: 271. (index mismatch)
I20240502 14:15:38.387977 2149 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240502 14:15:38.389010 30831 consensus_queue.cc:237] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 226, Committed index: 226, Last appended: 1.226, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 227 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:38.392904 1499 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 226. Preceding OpId from leader: term: 1 index: 227. (index mismatch)
I20240502 14:15:38.393913 30959 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 226. Preceding OpId from leader: term: 1 index: 227. (index mismatch)
I20240502 14:15:38.394582 31090 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 226. Preceding OpId from leader: term: 1 index: 227. (index mismatch)
I20240502 14:15:38.395037 32157 raft_consensus.cc:1261] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 155f26c73dc84441a58417c09f577494: Log matching property violated. Preceding OpId in replica: term: 1 index: 226. Preceding OpId from leader: term: 1 index: 227. (index mismatch)
I20240502 14:15:38.395228 31223 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 1 index: 271. (index mismatch)
I20240502 14:15:38.395522 2191 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 227, Last known committed idx: 226, Time since last communication: 0.000s
I20240502 14:15:38.395918 2021 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 227, Last known committed idx: 226, Time since last communication: 0.000s
I20240502 14:15:38.396291 2149 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240502 14:15:38.396629 2191 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 227, Last known committed idx: 226, Time since last communication: 0.000s
I20240502 14:15:38.397040 2028 consensus_queue.cc:1035] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 227, Last known committed idx: 226, Time since last communication: 0.000s
I20240502 14:15:38.400991 2028 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 [term 1 LEADER]: Committing config change with OpId 1.227: config changed from index 226 to 227, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 227 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.401608 1499 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.227: config changed from index 226 to 227, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 227 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.405680 30417 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0f906c7d775540e6ba76cf2ff3700a67 with cas_config_opid_index 226: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:38.405926 32157 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.227: config changed from index 226 to 227, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 227 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.406648 31090 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 270. Preceding OpId from leader: term: 1 index: 271. (index mismatch)
I20240502 14:15:38.408092 30429 catalog_manager.cc:5484] T 0f906c7d775540e6ba76cf2ff3700a67 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 226 to 227, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "155f26c73dc84441a58417c09f577494" committed_config { opid_index: 227 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:38.411863 30959 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.227: config changed from index 226 to 227, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 227 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.414083 31090 raft_consensus.cc:2932] T 0f906c7d775540e6ba76cf2ff3700a67 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.227: config changed from index 226 to 227, VOTER d733c51902044956a8cd2b6b31bc9831 (127.27.220.130) evicted. New config: { opid_index: 227 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.417042 2175 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 271, Last known committed idx: 270, Time since last communication: 0.000s
I20240502 14:15:38.417227 30682 tablet_service.cc:1514] Processing DeleteTablet for tablet 0f906c7d775540e6ba76cf2ff3700a67 with delete_type TABLET_DATA_TOMBSTONED (TS d733c51902044956a8cd2b6b31bc9831 not found in new config with opid_index 227) from {username='slave'} at 127.0.0.1:39850
I20240502 14:15:38.438012 2149 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.271: config changed from index 270 to 271, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 271 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.442899 31222 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.271: config changed from index 270 to 271, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 271 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.444475 31091 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.271: config changed from index 270 to 271, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 271 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.449317 30572 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.271: config changed from index 270 to 271, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 271 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.452108 30433 catalog_manager.cc:5484] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 reported cstate change: config changed from index 270 to 271, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 271 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.455911 1501 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.271: config changed from index 270 to 271, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 271 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.456902 32156 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.271: config changed from index 270 to 271, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 271 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } attrs { replace: true } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.466403 2210 tablet_replica.cc:330] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: stopping tablet replica
I20240502 14:15:38.466723 2210 raft_consensus.cc:2229] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:38.467020 2210 raft_consensus.cc:2258] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:38.470136 2210 ts_tablet_manager.cc:1875] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:38.474124 30959 consensus_queue.cc:237] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 271, Committed index: 271, Last appended: 1.271, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 272 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:38.477818 32156 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 271. Preceding OpId from leader: term: 1 index: 272. (index mismatch)
I20240502 14:15:38.477850 31222 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 271. Preceding OpId from leader: term: 1 index: 272. (index mismatch)
I20240502 14:15:38.478432 1501 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 271. Preceding OpId from leader: term: 1 index: 272. (index mismatch)
I20240502 14:15:38.478385 2011 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 272, Last known committed idx: 271, Time since last communication: 0.000s
I20240502 14:15:38.478694 2175 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 272, Last known committed idx: 271, Time since last communication: 0.000s
I20240502 14:15:38.479025 2149 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 272, Last known committed idx: 271, Time since last communication: 0.000s
I20240502 14:15:38.479198 30572 raft_consensus.cc:1261] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 271. Preceding OpId from leader: term: 1 index: 272. (index mismatch)
I20240502 14:15:38.479776 2212 consensus_queue.cc:1035] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 272, Last known committed idx: 271, Time since last communication: 0.000s
I20240502 14:15:38.481786 2210 ts_tablet_manager.cc:1888] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.226
I20240502 14:15:38.481959 2210 log.cc:1196] T 0f906c7d775540e6ba76cf2ff3700a67 P d733c51902044956a8cd2b6b31bc9831: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/0f906c7d775540e6ba76cf2ff3700a67
I20240502 14:15:38.482911 2175 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.272: config changed from index 271 to 272, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 272 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.483166 32156 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.272: config changed from index 271 to 272, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 272 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.484144 1501 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.272: config changed from index 271 to 272, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 272 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.484910 30572 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.272: config changed from index 271 to 272, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 272 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.486531 30418 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 50f7b19e515e497c9a27e8e94cb73079 with cas_config_opid_index 271: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:38.489264 30417 catalog_manager.cc:4830] TS d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): tablet 0f906c7d775540e6ba76cf2ff3700a67 (table rebalance_test_table_1 [id=68eff6e9fd9a4bf0a49dc0d8c6a7ef37]) successfully deleted
I20240502 14:15:38.489305 30433 catalog_manager.cc:5484] T 50f7b19e515e497c9a27e8e94cb73079 P 3eafb09239024de3bc0eea8d700f7668 reported cstate change: config changed from index 271 to 272, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 272 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.494307 31072 tablet_service.cc:1514] Processing DeleteTablet for tablet 50f7b19e515e497c9a27e8e94cb73079 with delete_type TABLET_DATA_TOMBSTONED (TS ba420f2442d6416bbda95edfff85e849 not found in new config with opid_index 272) from {username='slave'} at 127.0.0.1:42598
I20240502 14:15:38.496666 31222 raft_consensus.cc:2932] T 50f7b19e515e497c9a27e8e94cb73079 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Committing config change with OpId 1.272: config changed from index 271 to 272, VOTER ba420f2442d6416bbda95edfff85e849 (127.27.220.133) evicted. New config: { opid_index: 272 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.512887 2222 tablet_replica.cc:330] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: stopping tablet replica
I20240502 14:15:38.513182 2222 raft_consensus.cc:2229] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:38.513602 2222 raft_consensus.cc:2258] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:38.516346 2222 ts_tablet_manager.cc:1875] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:38.523492 2222 ts_tablet_manager.cc:1888] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.271
I20240502 14:15:38.523610 2222 log.cc:1196] T 50f7b19e515e497c9a27e8e94cb73079 P ba420f2442d6416bbda95edfff85e849: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/wals/50f7b19e515e497c9a27e8e94cb73079
I20240502 14:15:38.524354 30418 catalog_manager.cc:4830] TS ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805): tablet 50f7b19e515e497c9a27e8e94cb73079 (table rebalance_test_table_3 [id=f34c1def357a462cbceba8c68b410fe8]) successfully deleted
I20240502 14:15:38.627103 1501 raft_consensus.cc:1203] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.416->[1.417-1.417] Dedup: 1.417->[]
I20240502 14:15:38.655056 1943 raft_consensus.cc:1050] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:38.655840 1943 consensus_queue.cc:237] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 293, Committed index: 293, Last appended: 1.293, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:38.658366 30959 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 293. Preceding OpId from leader: term: 1 index: 294. (index mismatch)
I20240502 14:15:38.658715 32156 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 293. Preceding OpId from leader: term: 1 index: 294. (index mismatch)
I20240502 14:15:38.658880 30831 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 293. Preceding OpId from leader: term: 1 index: 294. (index mismatch)
I20240502 14:15:38.659312 1501 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 293. Preceding OpId from leader: term: 1 index: 294. (index mismatch)
I20240502 14:15:38.659250 2249 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 294, Last known committed idx: 293, Time since last communication: 0.000s
I20240502 14:15:38.659821 2069 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 294, Last known committed idx: 293, Time since last communication: 0.000s
I20240502 14:15:38.660161 2179 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 294, Last known committed idx: 293, Time since last communication: 0.000s
I20240502 14:15:38.661286 30698 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 293. Preceding OpId from leader: term: 1 index: 294. (index mismatch)
I20240502 14:15:38.661586 2183 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 294, Last known committed idx: 293, Time since last communication: 0.000s
I20240502 14:15:38.662508 2179 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 294, Last known committed idx: 293, Time since last communication: 0.000s
I20240502 14:15:38.671599 2069 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Committing config change with OpId 1.294: config changed from index 293 to 294, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.672719 1501 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.294: config changed from index 293 to 294, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.673651 30698 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Committing config change with OpId 1.294: config changed from index 293 to 294, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.676358 30959 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Committing config change with OpId 1.294: config changed from index 293 to 294, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.676625 32156 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.294: config changed from index 293 to 294, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.676970 30831 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.294: config changed from index 293 to 294, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.678032 30433 catalog_manager.cc:5484] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a reported cstate change: config changed from index 293 to 294, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:38.786721 31222 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13"
dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42178
I20240502 14:15:38.787034 31222 raft_consensus.cc:595] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Received request to transfer leadership
I20240502 14:15:39.014058 31222 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13"
dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42190
I20240502 14:15:39.014273 31222 raft_consensus.cc:595] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Received request to transfer leadership
I20240502 14:15:39.085875 2179 raft_consensus.cc:979] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: : Instructing follower 0f4f2e4ef00d4a648f1ccb69de68771a to start an election
I20240502 14:15:39.086079 2179 raft_consensus.cc:1067] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Signalling peer 0f4f2e4ef00d4a648f1ccb69de68771a to start an election
I20240502 14:15:39.086925 1501 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13"
dest_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a"
from {username='slave'} at 127.27.220.134:36439
I20240502 14:15:39.087229 1501 raft_consensus.cc:482] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 14:15:39.087374 1501 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:39.090052 1501 raft_consensus.cc:504] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:39.092108 1501 leader_election.cc:288] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 election: Requested vote from peers 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091)
I20240502 14:15:39.092908 31222 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 294 } } ignore_live_leader: true dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f"
I20240502 14:15:39.093184 31222 raft_consensus.cc:3032] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:15:39.093281 31222 raft_consensus.cc:729] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
I20240502 14:15:39.093660 31222 consensus_queue.cc:260] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 294, Committed index: 294, Last appended: 1.294, Last appended by leader: 294, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:39.093770 30961 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 294 } } ignore_live_leader: true dest_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3"
I20240502 14:15:39.094249 30961 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:39.094395 31222 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:39.097767 30831 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 294 } } ignore_live_leader: true dest_uuid: "155f26c73dc84441a58417c09f577494"
I20240502 14:15:39.098026 30831 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:39.099063 31222 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f4f2e4ef00d4a648f1ccb69de68771a in term 2.
I20240502 14:15:39.101747 30961 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f4f2e4ef00d4a648f1ccb69de68771a in term 2.
I20240502 14:15:39.104043 30831 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f4f2e4ef00d4a648f1ccb69de68771a in term 2.
I20240502 14:15:39.105567 1433 leader_election.cc:302] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 6 voters: 4 yes votes; 0 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 0f4f2e4ef00d4a648f1ccb69de68771a, 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3; no voters:
I20240502 14:15:39.106055 1942 raft_consensus.cc:2781] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:15:39.106781 1942 raft_consensus.cc:686] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 2 LEADER]: Becoming Leader. State: Replica: 0f4f2e4ef00d4a648f1ccb69de68771a, State: Running, Role: LEADER
I20240502 14:15:39.107487 1942 consensus_queue.cc:237] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 294, Committed index: 294, Last appended: 1.294, Last appended by leader: 294, Current term: 2, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:39.111518 30698 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 294 } } ignore_live_leader: true dest_uuid: "d733c51902044956a8cd2b6b31bc9831"
I20240502 14:15:39.111769 30698 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:39.112931 32156 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "356c1606c56147a2868e60ab63c59c13" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 294 } } ignore_live_leader: true dest_uuid: "3eafb09239024de3bc0eea8d700f7668"
I20240502 14:15:39.113281 32156 raft_consensus.cc:3037] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:39.113741 30433 catalog_manager.cc:5484] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a reported cstate change: term changed from 1 to 2, leader changed from 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) to 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136). New cstate: current_term: 2 leader_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" committed_config { opid_index: 294 OBSOLETE_local: false peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:39.114497 30698 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f4f2e4ef00d4a648f1ccb69de68771a in term 2.
I20240502 14:15:39.115952 32156 raft_consensus.cc:2454] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0f4f2e4ef00d4a648f1ccb69de68771a in term 2.
I20240502 14:15:39.210052 2291 raft_consensus.cc:1050] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3: attempting to promote NON_VOTER 0f4f2e4ef00d4a648f1ccb69de68771a to VOTER
I20240502 14:15:39.210829 2291 consensus_queue.cc:237] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 417, Committed index: 417, Last appended: 1.417, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 418 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:39.213660 32156 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 417. Preceding OpId from leader: term: 1 index: 418. (index mismatch)
I20240502 14:15:39.214025 30572 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 417. Preceding OpId from leader: term: 1 index: 418. (index mismatch)
I20240502 14:15:39.214097 31094 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 417. Preceding OpId from leader: term: 1 index: 418. (index mismatch)
I20240502 14:15:39.214319 1501 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 LEARNER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 417. Preceding OpId from leader: term: 1 index: 418. (index mismatch)
I20240502 14:15:39.214761 2220 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 418, Last known committed idx: 417, Time since last communication: 0.000s
I20240502 14:15:39.215123 30831 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 417. Preceding OpId from leader: term: 1 index: 418. (index mismatch)
I20240502 14:15:39.215061 2149 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 418, Last known committed idx: 417, Time since last communication: 0.000s
I20240502 14:15:39.215415 2293 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 418, Last known committed idx: 417, Time since last communication: 0.000s
I20240502 14:15:39.215680 2291 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 418, Last known committed idx: 417, Time since last communication: 0.000s
I20240502 14:15:39.216127 2294 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 418, Last known committed idx: 417, Time since last communication: 0.000s
I20240502 14:15:39.219841 2149 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.418: config changed from index 417 to 418, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 418 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.220502 30831 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Committing config change with OpId 1.418: config changed from index 417 to 418, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 418 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.228248 1501 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.418: config changed from index 417 to 418, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 418 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.228911 30433 catalog_manager.cc:5484] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 reported cstate change: config changed from index 417 to 418, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 418 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.231251 31094 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.418: config changed from index 417 to 418, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 418 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.239908 30572 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.418: config changed from index 417 to 418, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 418 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.240809 32156 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.418: config changed from index 417 to 418, 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 418 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } attrs { replace: true } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.243521 30961 consensus_queue.cc:237] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 418, Committed index: 418, Last appended: 1.418, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 419 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:39.246234 30570 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 418. Preceding OpId from leader: term: 1 index: 419. (index mismatch)
I20240502 14:15:39.246968 2149 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 419, Last known committed idx: 418, Time since last communication: 0.000s
I20240502 14:15:39.251391 32157 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 418. Preceding OpId from leader: term: 1 index: 419. (index mismatch)
I20240502 14:15:39.253208 2291 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 419, Last known committed idx: 418, Time since last communication: 0.000s
I20240502 14:15:39.255939 2149 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 1 LEADER]: Committing config change with OpId 1.419: config changed from index 418 to 419, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 419 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.256831 32157 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Committing config change with OpId 1.419: config changed from index 418 to 419, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 419 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.260255 30418 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 74c46ec356414d26987e1cd6a0857602 with cas_config_opid_index 418: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:39.262811 31091 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 418. Preceding OpId from leader: term: 1 index: 419. (index mismatch)
I20240502 14:15:39.263832 2149 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 419, Last known committed idx: 418, Time since last communication: 0.000s
I20240502 14:15:39.264686 30427 catalog_manager.cc:5484] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 reported cstate change: config changed from index 418 to 419, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" committed_config { opid_index: 419 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:15:39.264791 30572 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P fd3399b1bfe644159a833ede6726c25f [term 1 FOLLOWER]: Committing config change with OpId 1.419: config changed from index 418 to 419, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 419 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.266800 31091 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Committing config change with OpId 1.419: config changed from index 418 to 419, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 419 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.274119 30813 tablet_service.cc:1514] Processing DeleteTablet for tablet 74c46ec356414d26987e1cd6a0857602 with delete_type TABLET_DATA_TOMBSTONED (TS 155f26c73dc84441a58417c09f577494 not found in new config with opid_index 419) from {username='slave'} at 127.0.0.1:33764
I20240502 14:15:39.274971 2303 tablet_replica.cc:330] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: stopping tablet replica
I20240502 14:15:39.275485 2303 raft_consensus.cc:2229] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:39.276085 2303 raft_consensus.cc:2258] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:39.277160 1499 raft_consensus.cc:1261] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Refusing update from remote peer 7c1f568f8efc4b2a9dbf38907f02f3e3: Log matching property violated. Preceding OpId in replica: term: 1 index: 418. Preceding OpId from leader: term: 1 index: 419. (index mismatch)
I20240502 14:15:39.278051 2294 consensus_queue.cc:1035] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 419, Last known committed idx: 418, Time since last communication: 0.000s
I20240502 14:15:39.280328 1501 raft_consensus.cc:2932] T 74c46ec356414d26987e1cd6a0857602 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Committing config change with OpId 1.419: config changed from index 418 to 419, VOTER 155f26c73dc84441a58417c09f577494 (127.27.220.131) evicted. New config: { opid_index: 419 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:39.286365 2303 ts_tablet_manager.cc:1875] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:39.295154 2303 ts_tablet_manager.cc:1888] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.418
I20240502 14:15:39.295469 2303 log.cc:1196] T 74c46ec356414d26987e1cd6a0857602 P 155f26c73dc84441a58417c09f577494: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/74c46ec356414d26987e1cd6a0857602
I20240502 14:15:39.296653 30417 catalog_manager.cc:4830] TS 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): tablet 74c46ec356414d26987e1cd6a0857602 (table rebalance_test_table_0 [id=8fcd7c87840d4c05a7c9add28662b083]) successfully deleted
I20240502 14:15:39.520359 31222 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 1 index: 294. Preceding OpId from leader: term: 2 index: 295. (index mismatch)
I20240502 14:15:39.521164 1942 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 295, Last known committed idx: 294, Time since last communication: 0.000s
I20240502 14:15:39.522933 30831 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 1 index: 294. Preceding OpId from leader: term: 2 index: 295. (index mismatch)
I20240502 14:15:39.523561 1942 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 295, Last known committed idx: 294, Time since last communication: 0.000s
I20240502 14:15:39.575527 32157 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 1 index: 294. Preceding OpId from leader: term: 2 index: 295. (index mismatch)
I20240502 14:15:39.576037 2290 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 295, Last known committed idx: 294, Time since last communication: 0.000s
I20240502 14:15:39.581338 30961 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 1 index: 294. Preceding OpId from leader: term: 2 index: 295. (index mismatch)
I20240502 14:15:39.581902 2290 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 295, Last known committed idx: 294, Time since last communication: 0.000s
I20240502 14:15:39.590767 30698 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 1 index: 294. Preceding OpId from leader: term: 2 index: 295. (index mismatch)
I20240502 14:15:39.591472 2327 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 295, Last known committed idx: 294, Time since last communication: 0.000s
I20240502 14:15:40.191843 1501 consensus_queue.cc:237] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 295, Committed index: 295, Last appended: 2.295, Last appended by leader: 294, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 296 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:40.194888 32157 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 2 index: 295. Preceding OpId from leader: term: 2 index: 296. (index mismatch)
I20240502 14:15:40.195051 30961 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 2 index: 295. Preceding OpId from leader: term: 2 index: 296. (index mismatch)
I20240502 14:15:40.195622 1942 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 296, Last known committed idx: 295, Time since last communication: 0.000s
I20240502 14:15:40.195767 30831 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 2 index: 295. Preceding OpId from leader: term: 2 index: 296. (index mismatch)
I20240502 14:15:40.195838 30698 raft_consensus.cc:1261] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 2 FOLLOWER]: Refusing update from remote peer 0f4f2e4ef00d4a648f1ccb69de68771a: Log matching property violated. Preceding OpId in replica: term: 2 index: 295. Preceding OpId from leader: term: 2 index: 296. (index mismatch)
I20240502 14:15:40.196219 1942 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 296, Last known committed idx: 295, Time since last communication: 0.000s
I20240502 14:15:40.196553 2325 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 296, Last known committed idx: 295, Time since last communication: 0.000s
I20240502 14:15:40.196914 2327 consensus_queue.cc:1035] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [LEADER]: Connected to new peer: Peer: permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 296, Last known committed idx: 295, Time since last communication: 0.000s
I20240502 14:15:40.199129 1942 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 2 LEADER]: Committing config change with OpId 2.296: config changed from index 294 to 296, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 296 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:40.199501 30831 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 155f26c73dc84441a58417c09f577494 [term 2 FOLLOWER]: Committing config change with OpId 2.296: config changed from index 294 to 296, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 296 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:40.202539 32157 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 3eafb09239024de3bc0eea8d700f7668 [term 2 FOLLOWER]: Committing config change with OpId 2.296: config changed from index 294 to 296, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 296 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:40.204569 30419 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 356c1606c56147a2868e60ab63c59c13 with cas_config_opid_index 294: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:15:40.204685 30961 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P 7c1f568f8efc4b2a9dbf38907f02f3e3 [term 2 FOLLOWER]: Committing config change with OpId 2.296: config changed from index 294 to 296, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 296 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:40.205873 30426 catalog_manager.cc:5484] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a reported cstate change: config changed from index 294 to 296, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New cstate: current_term: 2 leader_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" committed_config { opid_index: 296 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:15:40.206375 30698 raft_consensus.cc:2932] T 356c1606c56147a2868e60ab63c59c13 P d733c51902044956a8cd2b6b31bc9831 [term 2 FOLLOWER]: Committing config change with OpId 2.296: config changed from index 294 to 296, VOTER 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134) evicted. New config: { opid_index: 296 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } } }
I20240502 14:15:40.227734 31204 tablet_service.cc:1514] Processing DeleteTablet for tablet 356c1606c56147a2868e60ab63c59c13 with delete_type TABLET_DATA_TOMBSTONED (TS 0dfd48d0b03a42f689e58d5190d03a2f not found in new config with opid_index 296) from {username='slave'} at 127.0.0.1:56766
I20240502 14:15:40.229930 2360 tablet_replica.cc:330] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: stopping tablet replica
I20240502 14:15:40.230232 2360 raft_consensus.cc:2229] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:15:40.230453 2360 raft_consensus.cc:2258] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:15:40.234372 2360 ts_tablet_manager.cc:1875] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:15:40.238629 2360 ts_tablet_manager.cc:1888] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.295
I20240502 14:15:40.238724 2360 log.cc:1196] T 356c1606c56147a2868e60ab63c59c13 P 0dfd48d0b03a42f689e58d5190d03a2f: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/wals/356c1606c56147a2868e60ab63c59c13
I20240502 14:15:40.239456 30420 catalog_manager.cc:4830] TS 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403): tablet 356c1606c56147a2868e60ab63c59c13 (table rebalance_test_table_2 [id=2feacfcde46a4cb1a985df105a1fb8bd]) successfully deleted
I20240502 14:15:40.351142 1481 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:15:40.373778 31204 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:15:40.421697 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:40.433729 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:40.438545 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:40.466030 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:40.493801 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:40.500312 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:40.678702 1481 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:15:40.683938 31204 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:15:40.706315 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:40.737010 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:40.737618 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:40.762419 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:40.763119 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:40.774294 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:41.129704 31204 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:15:41.138113 30813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:41.146456 30943 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:15:41.147809 1481 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:15:41.182502 32138 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:41.194980 30682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:15:41.200208 30553 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:15:41.209033 31072 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
1cb98ed728ea452ab7c2d0de3cb89c33 | 127.27.220.190:33157 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+----------------------+-------------------------
builtin_ntp_servers | 127.27.220.148:41401 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+----------------------+---------+----------+----------------+-----------------
0dfd48d0b03a42f689e58d5190d03a2f | 127.27.220.134:39403 | HEALTHY | <none> | 6 | 0
0f4f2e4ef00d4a648f1ccb69de68771a | 127.27.220.136:38139 | HEALTHY | <none> | 1 | 0
155f26c73dc84441a58417c09f577494 | 127.27.220.131:34975 | HEALTHY | <none> | 2 | 0
3eafb09239024de3bc0eea8d700f7668 | 127.27.220.135:33091 | HEALTHY | <none> | 0 | 0
7c1f568f8efc4b2a9dbf38907f02f3e3 | 127.27.220.132:41511 | HEALTHY | <none> | 3 | 0
ba420f2442d6416bbda95edfff85e849 | 127.27.220.133:34805 | HEALTHY | <none> | 2 | 0
d733c51902044956a8cd2b6b31bc9831 | 127.27.220.130:46259 | HEALTHY | <none> | 0 | 0
fd3399b1bfe644159a833ede6726c25f | 127.27.220.129:33055 | HEALTHY | <none> | 2 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 8
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 8 server(s) checked
local_ip_for_outbound_sockets | 127.27.220.129 | experimental | 127.27.220.129:33055
local_ip_for_outbound_sockets | 127.27.220.130 | experimental | 127.27.220.130:46259
local_ip_for_outbound_sockets | 127.27.220.131 | experimental | 127.27.220.131:34975
local_ip_for_outbound_sockets | 127.27.220.132 | experimental | 127.27.220.132:41511
local_ip_for_outbound_sockets | 127.27.220.133 | experimental | 127.27.220.133:34805
local_ip_for_outbound_sockets | 127.27.220.134 | experimental | 127.27.220.134:39403
local_ip_for_outbound_sockets | 127.27.220.135 | experimental | 127.27.220.135:33091
local_ip_for_outbound_sockets | 127.27.220.136 | experimental | 127.27.220.136:38139
never_fsync | true | unsafe,advanced | all 8 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 8 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 8 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 8 server(s) checked
server_dump_info_format | pb | hidden | all 8 server(s) checked
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.27.220.129:33055
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.27.220.130:46259
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.27.220.131:34975
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.27.220.132:41511
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.27.220.133:34805
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.27.220.134:39403
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.27.220.135:33091
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb | hidden | 127.27.220.136:38139
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+----------------------+-------------------------
builtin_ntp_servers | 127.27.220.148:41401 | all 8 server(s) checked
time_source | builtin | all 8 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 9 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 5 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 10
First Quartile | 10
Median | 10
Third Quartile | 10
Maximum | 10
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 8
Tables | 6
Tablets | 16
Replicas | 80
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 14:15:41.314023 28530 log_verifier.cc:126] Checking tablet 0f906c7d775540e6ba76cf2ff3700a67
I20240502 14:15:41.578994 28530 log_verifier.cc:177] Verified matching terms for 227 ops in tablet 0f906c7d775540e6ba76cf2ff3700a67
I20240502 14:15:41.579576 28530 log_verifier.cc:126] Checking tablet 23cf772ace1c44ecbd450e0bce820c3f
I20240502 14:15:41.761121 28530 log_verifier.cc:177] Verified matching terms for 188 ops in tablet 23cf772ace1c44ecbd450e0bce820c3f
I20240502 14:15:41.761742 28530 log_verifier.cc:126] Checking tablet 3183d887deec4e40b88dceda3306c0f1
I20240502 14:15:41.997594 28530 log_verifier.cc:177] Verified matching terms for 247 ops in tablet 3183d887deec4e40b88dceda3306c0f1
I20240502 14:15:41.998190 28530 log_verifier.cc:126] Checking tablet 356c1606c56147a2868e60ab63c59c13
I20240502 14:15:42.272704 28530 log_verifier.cc:177] Verified matching terms for 296 ops in tablet 356c1606c56147a2868e60ab63c59c13
I20240502 14:15:42.273648 28530 log_verifier.cc:126] Checking tablet 36b27125eebb410e8c37560450778dd5
I20240502 14:15:42.531666 28530 log_verifier.cc:177] Verified matching terms for 264 ops in tablet 36b27125eebb410e8c37560450778dd5
I20240502 14:15:42.532824 28530 log_verifier.cc:126] Checking tablet 3c26d346223a498bb96b962fa7dd99aa
I20240502 14:15:42.785934 28530 log_verifier.cc:177] Verified matching terms for 247 ops in tablet 3c26d346223a498bb96b962fa7dd99aa
I20240502 14:15:42.786724 28530 log_verifier.cc:126] Checking tablet 50f7b19e515e497c9a27e8e94cb73079
I20240502 14:15:43.030228 28530 log_verifier.cc:177] Verified matching terms for 272 ops in tablet 50f7b19e515e497c9a27e8e94cb73079
I20240502 14:15:43.030790 28530 log_verifier.cc:126] Checking tablet 5c4ceedcb00a49e1a32977e4e1acea24
I20240502 14:15:43.400748 28530 log_verifier.cc:177] Verified matching terms for 434 ops in tablet 5c4ceedcb00a49e1a32977e4e1acea24
I20240502 14:15:43.401670 28530 log_verifier.cc:126] Checking tablet 74c46ec356414d26987e1cd6a0857602
I20240502 14:15:43.797165 28530 log_verifier.cc:177] Verified matching terms for 419 ops in tablet 74c46ec356414d26987e1cd6a0857602
I20240502 14:15:43.798159 28530 log_verifier.cc:126] Checking tablet 7e5aa36e1a914374807e91f44bb1cff9
I20240502 14:15:44.057189 28530 log_verifier.cc:177] Verified matching terms for 298 ops in tablet 7e5aa36e1a914374807e91f44bb1cff9
I20240502 14:15:44.058087 28530 log_verifier.cc:126] Checking tablet 8b1b5e519c314a83bf1e39e011440fa6
I20240502 14:15:44.352003 28530 log_verifier.cc:177] Verified matching terms for 305 ops in tablet 8b1b5e519c314a83bf1e39e011440fa6
I20240502 14:15:44.353333 28530 log_verifier.cc:126] Checking tablet 975ae0ae782a45aebda66d2e32713def
I20240502 14:15:44.655737 28530 log_verifier.cc:177] Verified matching terms for 383 ops in tablet 975ae0ae782a45aebda66d2e32713def
I20240502 14:15:44.656580 28530 log_verifier.cc:126] Checking tablet af7e809edf3d44739814918dc2bff263
I20240502 14:15:44.891497 28530 log_verifier.cc:177] Verified matching terms for 247 ops in tablet af7e809edf3d44739814918dc2bff263
I20240502 14:15:44.892141 28530 log_verifier.cc:126] Checking tablet b63847bdc0e546bca632a85255a45572
I20240502 14:15:44.928350 28530 log_verifier.cc:177] Verified matching terms for 7 ops in tablet b63847bdc0e546bca632a85255a45572
I20240502 14:15:44.928450 28530 log_verifier.cc:126] Checking tablet bfa09c15ce944ce29fce3bbc1ad4df56
I20240502 14:15:45.170495 28530 log_verifier.cc:177] Verified matching terms for 280 ops in tablet bfa09c15ce944ce29fce3bbc1ad4df56
I20240502 14:15:45.171243 28530 log_verifier.cc:126] Checking tablet d4aa1eb08d4b450f9c5be499d8c6bbc7
I20240502 14:15:45.410903 28530 log_verifier.cc:177] Verified matching terms for 239 ops in tablet d4aa1eb08d4b450f9c5be499d8c6bbc7
I20240502 14:15:45.416738 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 30492
I20240502 14:15:45.427182 30614 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:45.707003 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 30492
W20240502 14:15:45.758551 30899 connection.cc:535] client connection to 127.27.220.129:33055 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:15:45.758759 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 30622
W20240502 14:15:45.758857 30899 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:15:45.762632 30899 consensus_peers.cc:485] T 50f7b19e515e497c9a27e8e94cb73079 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Couldn't send request to peer fd3399b1bfe644159a833ede6726c25f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:45.766372 30743 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:15:45.830237 31029 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
W20240502 14:15:45.832149 31029 consensus_peers.cc:485] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 -> Peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Couldn't send request to peer fd3399b1bfe644159a833ede6726c25f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:45.842780 30899 consensus_peers.cc:485] T 74c46ec356414d26987e1cd6a0857602 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Couldn't send request to peer fd3399b1bfe644159a833ede6726c25f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:45.916036 31159 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
W20240502 14:15:45.917949 31159 consensus_peers.cc:485] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Couldn't send request to peer fd3399b1bfe644159a833ede6726c25f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:45.955072 30769 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
W20240502 14:15:45.956423 30769 consensus_peers.cc:485] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 -> Peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Couldn't send request to peer fd3399b1bfe644159a833ede6726c25f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.008124 31159 consensus_peers.cc:485] T 8b1b5e519c314a83bf1e39e011440fa6 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Couldn't send request to peer fd3399b1bfe644159a833ede6726c25f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:46.028270 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 30622
W20240502 14:15:46.075686 31027 connection.cc:535] client connection to 127.27.220.130:46259 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:15:46.076195 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 30752
W20240502 14:15:46.077404 31027 consensus_peers.cc:485] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.077775 30767 consensus_peers.cc:485] T 7e5aa36e1a914374807e91f44bb1cff9 P 155f26c73dc84441a58417c09f577494 -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:46.082547 30874 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:15:46.095119 31159 consensus_peers.cc:485] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Couldn't send request to peer fd3399b1bfe644159a833ede6726c25f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.123250 1433 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111)
W20240502 14:15:46.124806 1433 consensus_peers.cc:485] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.138638 31159 consensus_peers.cc:485] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Couldn't send request to peer fd3399b1bfe644159a833ede6726c25f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.203795 31157 consensus_peers.cc:485] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.231264 31157 consensus_peers.cc:485] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.261005 30897 consensus_peers.cc:485] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:46.357033 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 30752
I20240502 14:15:46.359664 2501 raft_consensus.cc:482] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fd3399b1bfe644159a833ede6726c25f)
I20240502 14:15:46.360023 2501 raft_consensus.cc:504] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:46.362174 2501 leader_election.cc:288] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139)
I20240502 14:15:46.362938 31222 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "3eafb09239024de3bc0eea8d700f7668" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:15:46.363497 31091 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "3eafb09239024de3bc0eea8d700f7668" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
W20240502 14:15:46.369732 32093 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
W20240502 14:15:46.372130 32093 leader_election.cc:334] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
I20240502 14:15:46.372285 32093 leader_election.cc:302] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 3eafb09239024de3bc0eea8d700f7668; no voters: 0dfd48d0b03a42f689e58d5190d03a2f, ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f
I20240502 14:15:46.372613 2501 raft_consensus.cc:2726] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:15:46.376235 1499 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "3eafb09239024de3bc0eea8d700f7668" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" is_pre_election: true
W20240502 14:15:46.409947 32094 connection.cc:535] server connection from 127.27.220.131:52377 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:15:46.410104 31027 connection.cc:535] client connection to 127.27.220.131:34975 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:15:46.410138 30897 connection.cc:535] client connection to 127.27.220.131:34975 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:15:46.410876 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 30882
W20240502 14:15:46.410955 31157 connection.cc:535] client connection to 127.27.220.131:34975 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:15:46.411638 30897 consensus_peers.cc:485] T 5c4ceedcb00a49e1a32977e4e1acea24 P 7c1f568f8efc4b2a9dbf38907f02f3e3 -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.411679 31027 consensus_peers.cc:485] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.412395 31157 consensus_peers.cc:485] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:46.417273 31003 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:15:46.438827 31027 consensus_peers.cc:485] T 36b27125eebb410e8c37560450778dd5 P ba420f2442d6416bbda95edfff85e849 -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.479931 31157 consensus_peers.cc:485] T 23cf772ace1c44ecbd450e0bce820c3f P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.558916 31157 consensus_peers.cc:485] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.572933 31157 consensus_peers.cc:485] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Couldn't send request to peer d733c51902044956a8cd2b6b31bc9831. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.669449 31027 consensus_peers.cc:485] T 3c26d346223a498bb96b962fa7dd99aa P ba420f2442d6416bbda95edfff85e849 -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:46.697199 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 30882
I20240502 14:15:46.719641 2508 raft_consensus.cc:482] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fd3399b1bfe644159a833ede6726c25f)
I20240502 14:15:46.719986 2508 raft_consensus.cc:504] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:46.721977 31091 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:15:46.723096 31222 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:15:46.723106 32157 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "3eafb09239024de3bc0eea8d700f7668" is_pre_election: true
I20240502 14:15:46.723371 32157 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f4f2e4ef00d4a648f1ccb69de68771a in term 1.
I20240502 14:15:46.723935 2508 leader_election.cc:288] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091)
W20240502 14:15:46.724001 1435 leader_election.cc:334] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
I20240502 14:15:46.724267 1436 leader_election.cc:302] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 5 voters: 2 yes votes; 3 no votes. yes voters: 0f4f2e4ef00d4a648f1ccb69de68771a, 3eafb09239024de3bc0eea8d700f7668; no voters: 0dfd48d0b03a42f689e58d5190d03a2f, ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f
I20240502 14:15:46.724586 2508 raft_consensus.cc:2726] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:15:46.729826 2479 raft_consensus.cc:482] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fd3399b1bfe644159a833ede6726c25f)
I20240502 14:15:46.729997 2479 raft_consensus.cc:504] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:46.731015 2479 leader_election.cc:288] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091), 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139)
I20240502 14:15:46.731343 31091 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
W20240502 14:15:46.731565 1433 consensus_peers.cc:485] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:46.732231 32157 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "3eafb09239024de3bc0eea8d700f7668" is_pre_election: true
I20240502 14:15:46.732461 32157 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:15:46.732426 1499 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" is_pre_election: true
I20240502 14:15:46.732687 1499 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
W20240502 14:15:46.733006 31159 leader_election.cc:334] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
I20240502 14:15:46.733296 31159 leader_election.cc:302] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 5 voters: 3 yes votes; 2 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 0f4f2e4ef00d4a648f1ccb69de68771a, 3eafb09239024de3bc0eea8d700f7668; no voters: ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f
I20240502 14:15:46.733565 2479 raft_consensus.cc:2781] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Leader pre-election won for term 2
I20240502 14:15:46.733645 2479 raft_consensus.cc:482] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting leader election (detected failure of leader fd3399b1bfe644159a833ede6726c25f)
I20240502 14:15:46.733716 2479 raft_consensus.cc:3037] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:46.735864 2479 raft_consensus.cc:504] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 2 FOLLOWER]: Starting leader election with config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:46.736596 2479 leader_election.cc:288] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 2 election: Requested vote from peers ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091), 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139)
I20240502 14:15:46.736838 31091 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849"
I20240502 14:15:46.737546 1499 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a"
I20240502 14:15:46.737761 1499 raft_consensus.cc:3037] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:46.737874 32157 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "3eafb09239024de3bc0eea8d700f7668"
I20240502 14:15:46.738099 32157 raft_consensus.cc:3037] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:15:46.740012 32157 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 2.
W20240502 14:15:46.740875 31159 leader_election.cc:334] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
I20240502 14:15:46.740933 1499 raft_consensus.cc:2454] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 2.
I20240502 14:15:46.741639 31157 leader_election.cc:302] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 5 voters: 3 yes votes; 2 no votes. yes voters: 0dfd48d0b03a42f689e58d5190d03a2f, 0f4f2e4ef00d4a648f1ccb69de68771a, 3eafb09239024de3bc0eea8d700f7668; no voters: ba420f2442d6416bbda95edfff85e849, fd3399b1bfe644159a833ede6726c25f
I20240502 14:15:46.741901 2479 raft_consensus.cc:2781] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:15:46.742038 2479 raft_consensus.cc:686] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [term 2 LEADER]: Becoming Leader. State: Replica: 0dfd48d0b03a42f689e58d5190d03a2f, State: Running, Role: LEADER
W20240502 14:15:46.742442 31157 consensus_peers.cc:485] T 975ae0ae782a45aebda66d2e32713def P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:46.742364 2479 consensus_queue.cc:237] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:46.746153 30433 catalog_manager.cc:5484] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f reported cstate change: term changed from 1 to 2, leader changed from fd3399b1bfe644159a833ede6726c25f (127.27.220.129) to 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134). New cstate: current_term: 2 leader_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
W20240502 14:15:46.751950 32092 connection.cc:535] server connection from 127.27.220.132:56155 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:15:46.753321 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 31012
I20240502 14:15:46.758801 31134 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:15:46.783319 31157 consensus_peers.cc:485] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Couldn't send request to peer 155f26c73dc84441a58417c09f577494. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:46.824596 2508 raft_consensus.cc:482] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fd3399b1bfe644159a833ede6726c25f)
I20240502 14:15:46.824842 2508 raft_consensus.cc:504] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:46.825948 2508 leader_election.cc:288] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805)
I20240502 14:15:46.827454 31222 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 247 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
W20240502 14:15:46.827760 1433 leader_election.cc:334] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111)
W20240502 14:15:46.828495 1435 leader_election.cc:334] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055): Network error: Client connection negotiation failed: client connection to 127.27.220.129:33055: connect: Connection refused (error 111)
I20240502 14:15:46.828649 1435 leader_election.cc:302] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 0f4f2e4ef00d4a648f1ccb69de68771a; no voters: 0dfd48d0b03a42f689e58d5190d03a2f, d733c51902044956a8cd2b6b31bc9831, fd3399b1bfe644159a833ede6726c25f
I20240502 14:15:46.828895 2508 raft_consensus.cc:2726] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20240502 14:15:46.892143 1434 consensus_peers.cc:485] T 356c1606c56147a2868e60ab63c59c13 P 0f4f2e4ef00d4a648f1ccb69de68771a -> Peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): Couldn't send request to peer 7c1f568f8efc4b2a9dbf38907f02f3e3. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.132:41511: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:46.920473 31158 consensus_peers.cc:485] T d4aa1eb08d4b450f9c5be499d8c6bbc7 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): Couldn't send request to peer 7c1f568f8efc4b2a9dbf38907f02f3e3. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.132:41511: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:47.020061 2520 raft_consensus.cc:482] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fd3399b1bfe644159a833ede6726c25f)
I20240502 14:15:47.020342 2520 raft_consensus.cc:504] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:47.021463 2520 leader_election.cc:288] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139)
I20240502 14:15:47.021672 2520 raft_consensus.cc:482] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fd3399b1bfe644159a833ede6726c25f)
I20240502 14:15:47.021865 2520 raft_consensus.cc:504] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:47.023013 2520 leader_election.cc:288] T b63847bdc0e546bca632a85255a45572 P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), 0dfd48d0b03a42f689e58d5190d03a2f (127.27.220.134:39403), 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091), 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139)
W20240502 14:15:47.023386 31027 leader_election.cc:334] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259): Network error: Client connection negotiation failed: client connection to 127.27.220.130:46259: connect: Connection refused (error 111)
I20240502 14:15:47.023640 32157 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "3eafb09239024de3bc0eea8d700f7668" is_pre_election: true
I20240502 14:15:47.023954 32157 raft_consensus.cc:2379] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ba420f2442d6416bbda95edfff85e849 in current term 2: Already voted for candidate 0dfd48d0b03a42f689e58d5190d03a2f in this term.
W20240502 14:15:47.030824 31158 consensus_peers.cc:485] T 3183d887deec4e40b88dceda3306c0f1 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): Couldn't send request to peer 7c1f568f8efc4b2a9dbf38907f02f3e3. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.132:41511: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:15:47.032653 31222 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 2 candidate_status { last_received { term: 1 index: 247 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:15:47.033078 31223 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b63847bdc0e546bca632a85255a45572" candidate_uuid: "ba420f2442d6416bbda95edfff85e849" candidate_term: 2 candidate_status { last_received { term: 1 index: 7 } } ignore_live_leader: false dest_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" is_pre_election: true
I20240502 14:15:47.033294 31091 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 247 } } ignore_live_leader: false dest_uuid: "ba420f2442d6416bbda95edfff85e849" is_pre_election: true
I20240502 14:15:47.033548 31091 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P ba420f2442d6416bbda95edfff85e849 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0f4f2e4ef00d4a648f1ccb69de68771a in term 1.
I20240502 14:15:47.034054 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 31012
W20240502 14:15:47.064221 31158 consensus_peers.cc:485] T bfa09c15ce944ce29fce3bbc1ad4df56 P 0dfd48d0b03a42f689e58d5190d03a2f -> Peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): Couldn't send request to peer 7c1f568f8efc4b2a9dbf38907f02f3e3. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.132:41511: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:15:47.085057 32092 connection.cc:535] server connection from 127.27.220.133:60039 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:15:47.085490 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 31142
W20240502 14:15:47.085532 30420 connection.cc:535] server connection from 127.27.220.133:39523 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:15:47.090762 31263 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:47.380357 2528 raft_consensus.cc:482] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting pre-election (detected failure of leader fd3399b1bfe644159a833ede6726c25f)
I20240502 14:15:47.380646 2528 raft_consensus.cc:504] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 247 OBSOLETE_local: false peers { permanent_uuid: "d733c51902044956a8cd2b6b31bc9831" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 46259 } } peers { permanent_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 39403 } } peers { permanent_uuid: "fd3399b1bfe644159a833ede6726c25f" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 33055 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:47.381704 2528 leader_election.cc:288] T af7e809edf3d44739814918dc2bff263 P 0dfd48d0b03a42f689e58d5190d03a2f [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers d733c51902044956a8cd2b6b31bc9831 (127.27.220.130:46259), fd3399b1bfe644159a833ede6726c25f (127.27.220.129:33055), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 0f4f2e4ef00d4a648f1ccb69de68771a (127.27.220.136:38139)
I20240502 14:15:47.381903 1499 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 2 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 14:15:47.382905 2179 consensus_queue.cc:1035] T b63847bdc0e546bca632a85255a45572 P 0dfd48d0b03a42f689e58d5190d03a2f [LEADER]: Connected to new peer: Peer: permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:15:47.383114 1499 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "af7e809edf3d44739814918dc2bff263" candidate_uuid: "0dfd48d0b03a42f689e58d5190d03a2f" candidate_term: 2 candidate_status { last_received { term: 1 index: 247 } } ignore_live_leader: false dest_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" is_pre_election: true
I20240502 14:15:47.383329 1499 raft_consensus.cc:2454] T af7e809edf3d44739814918dc2bff263 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0dfd48d0b03a42f689e58d5190d03a2f in term 1.
I20240502 14:15:47.384894 32156 raft_consensus.cc:1261] T b63847bdc0e546bca632a85255a45572 P 3eafb09239024de3bc0eea8d700f7668 [term 2 FOLLOWER]: Refusing update from remote peer 0dfd48d0b03a42f689e58d5190d03a2f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 14:15:47.385592 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 31142
W20240502 14:15:47.435488 32093 connection.cc:535] server connection from 127.27.220.134:48955 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:15:47.435536 1433 connection.cc:535] server connection from 127.27.220.134:36439 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:15:47.436131 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 32074
I20240502 14:15:47.441982 32197 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:47.460842 2526 raft_consensus.cc:482] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 155f26c73dc84441a58417c09f577494)
I20240502 14:15:47.461055 2526 raft_consensus.cc:504] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 227 OBSOLETE_local: false peers { permanent_uuid: "7c1f568f8efc4b2a9dbf38907f02f3e3" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 41511 } } peers { permanent_uuid: "ba420f2442d6416bbda95edfff85e849" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 34805 } } peers { permanent_uuid: "155f26c73dc84441a58417c09f577494" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 34975 } } peers { permanent_uuid: "3eafb09239024de3bc0eea8d700f7668" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 33091 } attrs { promote: false } } peers { permanent_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 38139 } attrs { promote: false } }
I20240502 14:15:47.462054 2526 leader_election.cc:288] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511), ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805), 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975), 3eafb09239024de3bc0eea8d700f7668 (127.27.220.135:33091)
W20240502 14:15:47.463816 1433 leader_election.cc:334] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 155f26c73dc84441a58417c09f577494 (127.27.220.131:34975): Network error: Client connection negotiation failed: client connection to 127.27.220.131:34975: connect: Connection refused (error 111)
W20240502 14:15:47.464224 1434 leader_election.cc:334] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 7c1f568f8efc4b2a9dbf38907f02f3e3 (127.27.220.132:41511): Network error: Client connection negotiation failed: client connection to 127.27.220.132:41511: connect: Connection refused (error 111)
W20240502 14:15:47.464416 1434 leader_election.cc:334] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer ba420f2442d6416bbda95edfff85e849 (127.27.220.133:34805): Network error: Client connection negotiation failed: client connection to 127.27.220.133:34805: connect: Connection refused (error 111)
I20240502 14:15:47.464524 1434 leader_election.cc:302] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 0f4f2e4ef00d4a648f1ccb69de68771a; no voters: 155f26c73dc84441a58417c09f577494, 7c1f568f8efc4b2a9dbf38907f02f3e3, ba420f2442d6416bbda95edfff85e849
I20240502 14:15:47.464802 2526 raft_consensus.cc:2726] T 0f906c7d775540e6ba76cf2ff3700a67 P 0f4f2e4ef00d4a648f1ccb69de68771a [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:15:47.669961 32156 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f906c7d775540e6ba76cf2ff3700a67" candidate_uuid: "0f4f2e4ef00d4a648f1ccb69de68771a" candidate_term: 2 candidate_status { last_received { term: 1 index: 227 } } ignore_live_leader: false dest_uuid: "3eafb09239024de3bc0eea8d700f7668" is_pre_election: true
I20240502 14:15:47.670521 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 32074
I20240502 14:15:47.711938 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 1417
I20240502 14:15:47.717691 1541 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:47.877048 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 1417
I20240502 14:15:47.904047 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 30402
I20240502 14:15:47.905114 30462 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:48.089613 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 30402
2024-05-02T14:15:48Z chronyd exiting
[ OK ] RebalanceParamTest.Rebalance/7 (77087 ms)
[----------] 1 test from RebalanceParamTest (77087 ms total)
[----------] 1 test from TserverAddedDuringRebalancingTest
[ RUN ] TserverAddedDuringRebalancingTest.TserverStarts/1
I20240502 14:15:48.246790 28530 test_util.cc:268] Using random seed: 1029501161
I20240502 14:15:48.248451 28530 ts_itest-base.cc:112] Starting cluster with:
I20240502 14:15:48.248509 28530 ts_itest-base.cc:113] --------------
I20240502 14:15:48.248553 28530 ts_itest-base.cc:114] 8 tablet servers
I20240502 14:15:48.248606 28530 ts_itest-base.cc:115] 3 replicas per TS
I20240502 14:15:48.248661 28530 ts_itest-base.cc:116] --------------
2024-05-02T14:15:48Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T14:15:48Z Disabled control of system clock
I20240502 14:15:48.303102 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:35583
--webserver_interface=127.27.220.190
--webserver_port=0
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.27.220.190:35583
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 14:15:48.645126 2550 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:48.645484 2550 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:48.645643 2550 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:48.653537 2550 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:48.653683 2550 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 14:15:48.653770 2550 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:48.653851 2550 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 14:15:48.653920 2550 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 14:15:48.663044 2550 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.27.220.190:35583
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:35583
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.27.220.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:48.664444 2550 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:48.666319 2550 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:48.677872 2556 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:48.678527 2555 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:48.679181 2558 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:48.680066 2550 server_base.cc:1034] running on GCE node
I20240502 14:15:48.680855 2550 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:48.682328 2550 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:48.683549 2550 hybrid_clock.cc:648] HybridClock initialized: now 1714659348683506 us; error 41 us; skew 500 ppm
I20240502 14:15:48.683905 2550 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:48.686249 2550 webserver.cc:460] Webserver started at http://127.27.220.190:41573/ using document root <none> and password file <none>
I20240502 14:15:48.686966 2550 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:48.687093 2550 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:48.687433 2550 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:48.690079 2550 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "5f080ba1a0a44cb0982d5c0d67b8633f"
format_stamp: "Formatted at 2024-05-02 14:15:48 on dist-test-slave-j72s"
I20240502 14:15:48.690826 2550 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "5f080ba1a0a44cb0982d5c0d67b8633f"
format_stamp: "Formatted at 2024-05-02 14:15:48 on dist-test-slave-j72s"
I20240502 14:15:48.695907 2550 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 14:15:48.699067 2564 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:48.700296 2550 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 14:15:48.700479 2550 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
uuid: "5f080ba1a0a44cb0982d5c0d67b8633f"
format_stamp: "Formatted at 2024-05-02 14:15:48 on dist-test-slave-j72s"
I20240502 14:15:48.700671 2550 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:48.725080 2550 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:48.725939 2550 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:48.726223 2550 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:48.744271 2550 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.190:35583
I20240502 14:15:48.744279 2615 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.190:35583 every 8 connection(s)
I20240502 14:15:48.745618 2550 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 14:15:48.749229 2616 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:48.750630 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 2550
I20240502 14:15:48.750851 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 14:15:48.761379 2616 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f: Bootstrap starting.
I20240502 14:15:48.765075 2616 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:48.766152 2616 log.cc:824] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f: Log is configured to *not* fsync() on all Append() calls
I20240502 14:15:48.768535 2616 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f: No bootstrap required, opened a new log
I20240502 14:15:48.774920 2616 raft_consensus.cc:348] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5f080ba1a0a44cb0982d5c0d67b8633f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 35583 } }
I20240502 14:15:48.775245 2616 raft_consensus.cc:374] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:48.775327 2616 raft_consensus.cc:729] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5f080ba1a0a44cb0982d5c0d67b8633f, State: Initialized, Role: FOLLOWER
I20240502 14:15:48.775902 2616 consensus_queue.cc:260] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5f080ba1a0a44cb0982d5c0d67b8633f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 35583 } }
I20240502 14:15:48.776171 2616 raft_consensus.cc:388] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:15:48.776361 2616 raft_consensus.cc:482] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:15:48.776535 2616 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:48.778750 2616 raft_consensus.cc:504] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5f080ba1a0a44cb0982d5c0d67b8633f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 35583 } }
I20240502 14:15:48.779210 2616 leader_election.cc:302] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 5f080ba1a0a44cb0982d5c0d67b8633f; no voters:
I20240502 14:15:48.779870 2616 leader_election.cc:288] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:15:48.779981 2621 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:48.780732 2621 raft_consensus.cc:686] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [term 1 LEADER]: Becoming Leader. State: Replica: 5f080ba1a0a44cb0982d5c0d67b8633f, State: Running, Role: LEADER
I20240502 14:15:48.781312 2621 consensus_queue.cc:237] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5f080ba1a0a44cb0982d5c0d67b8633f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 35583 } }
I20240502 14:15:48.781947 2616 sys_catalog.cc:564] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [sys.catalog]: configured and running, proceeding with master startup.
I20240502 14:15:48.784355 2623 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [sys.catalog]: SysCatalogTable state changed. Reason: New leader 5f080ba1a0a44cb0982d5c0d67b8633f. Latest consensus state: current_term: 1 leader_uuid: "5f080ba1a0a44cb0982d5c0d67b8633f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5f080ba1a0a44cb0982d5c0d67b8633f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 35583 } } }
I20240502 14:15:48.784866 2623 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [sys.catalog]: This master's current role is: LEADER
I20240502 14:15:48.785432 2622 sys_catalog.cc:455] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "5f080ba1a0a44cb0982d5c0d67b8633f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5f080ba1a0a44cb0982d5c0d67b8633f" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 35583 } } }
I20240502 14:15:48.785754 2622 sys_catalog.cc:458] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f [sys.catalog]: This master's current role is: LEADER
I20240502 14:15:48.786940 2629 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 14:15:48.791810 2629 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 14:15:48.799440 2629 catalog_manager.cc:1347] Generated new cluster ID: 3cc510ff4a9d4371b9710b1dddb3310e
I20240502 14:15:48.799548 2629 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 14:15:48.819142 2629 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 14:15:48.820811 2629 catalog_manager.cc:1504] Loading token signing keys...
I20240502 14:15:48.831604 2629 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 5f080ba1a0a44cb0982d5c0d67b8633f: Generated new TSK 0
I20240502 14:15:48.832618 2629 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 14:15:48.845314 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.129:0
--local_ip_for_outbound_sockets=127.27.220.129
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
W20240502 14:15:49.183657 2640 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:49.183959 2640 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:49.184060 2640 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:49.191529 2640 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:49.191642 2640 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:49.191767 2640 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.129
I20240502 14:15:49.200583 2640 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.27.220.129
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.129
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:49.201886 2640 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:49.203575 2640 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:49.215191 2645 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:49.215961 2646 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:49.216429 2648 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:49.217298 2640 server_base.cc:1034] running on GCE node
I20240502 14:15:49.217897 2640 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:49.218819 2640 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:49.220005 2640 hybrid_clock.cc:648] HybridClock initialized: now 1714659349219963 us; error 37 us; skew 500 ppm
I20240502 14:15:49.220278 2640 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:49.226622 2640 webserver.cc:460] Webserver started at http://127.27.220.129:39959/ using document root <none> and password file <none>
I20240502 14:15:49.227325 2640 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:49.227430 2640 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:49.227742 2640 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:49.230388 2640 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "6b3499bcb84b45a094f7166639704ed4"
format_stamp: "Formatted at 2024-05-02 14:15:49 on dist-test-slave-j72s"
I20240502 14:15:49.231106 2640 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "6b3499bcb84b45a094f7166639704ed4"
format_stamp: "Formatted at 2024-05-02 14:15:49 on dist-test-slave-j72s"
I20240502 14:15:49.236212 2640 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 14:15:49.239428 2654 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:49.240657 2640 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:49.240819 2640 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "6b3499bcb84b45a094f7166639704ed4"
format_stamp: "Formatted at 2024-05-02 14:15:49 on dist-test-slave-j72s"
I20240502 14:15:49.241024 2640 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:49.259729 2640 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:49.260526 2640 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:49.260793 2640 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:49.261762 2640 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:49.263214 2640 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:49.263307 2640 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:49.263417 2640 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:49.263485 2640 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:49.295167 2640 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.129:44787
I20240502 14:15:49.295235 2766 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.129:44787 every 8 connection(s)
I20240502 14:15:49.296525 2640 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 14:15:49.306900 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 2640
I20240502 14:15:49.307178 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 14:15:49.310643 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.130:0
--local_ip_for_outbound_sockets=127.27.220.130
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:15:49.310952 2767 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:15:49.311326 2767 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:49.312466 2767 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:15:49.315008 2581 ts_manager.cc:194] Registered new tserver with Master: 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:15:49.317250 2581 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.129:51571
W20240502 14:15:49.665695 2771 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:49.665997 2771 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:49.666205 2771 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:49.673576 2771 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:49.673692 2771 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:49.673851 2771 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.130
I20240502 14:15:49.682783 2771 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.27.220.130
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.130
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:49.684121 2771 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:49.685940 2771 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:49.698117 2776 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:49.698599 2779 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:49.699383 2777 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:49.700198 2771 server_base.cc:1034] running on GCE node
I20240502 14:15:49.700809 2771 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:49.701807 2771 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:49.703051 2771 hybrid_clock.cc:648] HybridClock initialized: now 1714659349702999 us; error 40 us; skew 500 ppm
I20240502 14:15:49.703378 2771 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:49.705644 2771 webserver.cc:460] Webserver started at http://127.27.220.130:44413/ using document root <none> and password file <none>
I20240502 14:15:49.706363 2771 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:49.706491 2771 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:49.706805 2771 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:49.709309 2771 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "c6abc8535fe24d03bf71fb7092072fc4"
format_stamp: "Formatted at 2024-05-02 14:15:49 on dist-test-slave-j72s"
I20240502 14:15:49.710103 2771 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "c6abc8535fe24d03bf71fb7092072fc4"
format_stamp: "Formatted at 2024-05-02 14:15:49 on dist-test-slave-j72s"
I20240502 14:15:49.715116 2771 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240502 14:15:49.718442 2785 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:49.719749 2771 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 14:15:49.719988 2771 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "c6abc8535fe24d03bf71fb7092072fc4"
format_stamp: "Formatted at 2024-05-02 14:15:49 on dist-test-slave-j72s"
I20240502 14:15:49.720198 2771 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:49.758069 2771 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:49.759094 2771 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:49.759397 2771 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:49.760704 2771 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:49.762805 2771 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:49.762923 2771 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:49.763028 2771 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:49.763098 2771 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:49.797149 2771 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.130:44843
I20240502 14:15:49.797201 2897 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.130:44843 every 8 connection(s)
I20240502 14:15:49.798697 2771 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 14:15:49.805495 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 2771
I20240502 14:15:49.805738 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 14:15:49.809012 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:0
--local_ip_for_outbound_sockets=127.27.220.131
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:15:49.812021 2898 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:15:49.812353 2898 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:49.813344 2898 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:15:49.814988 2581 ts_manager.cc:194] Registered new tserver with Master: c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:15:49.815830 2581 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.130:60371
W20240502 14:15:50.134943 2901 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:50.135226 2901 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:50.135378 2901 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:50.142704 2901 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:50.142824 2901 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:50.142987 2901 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:15:50.151932 2901 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:50.153223 2901 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:50.154899 2901 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:50.166858 2906 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:50.167346 2907 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:50.168084 2909 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:50.168745 2901 server_base.cc:1034] running on GCE node
I20240502 14:15:50.169456 2901 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:50.170495 2901 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:50.171698 2901 hybrid_clock.cc:648] HybridClock initialized: now 1714659350171650 us; error 37 us; skew 500 ppm
I20240502 14:15:50.172052 2901 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:50.174423 2901 webserver.cc:460] Webserver started at http://127.27.220.131:46155/ using document root <none> and password file <none>
I20240502 14:15:50.175171 2901 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:50.175302 2901 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:50.175647 2901 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:50.178378 2901 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "8cee803d912546e1981521fd8f16bb4e"
format_stamp: "Formatted at 2024-05-02 14:15:50 on dist-test-slave-j72s"
I20240502 14:15:50.179107 2901 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "8cee803d912546e1981521fd8f16bb4e"
format_stamp: "Formatted at 2024-05-02 14:15:50 on dist-test-slave-j72s"
I20240502 14:15:50.184147 2901 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240502 14:15:50.187420 2915 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:50.188623 2901 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:50.188783 2901 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "8cee803d912546e1981521fd8f16bb4e"
format_stamp: "Formatted at 2024-05-02 14:15:50 on dist-test-slave-j72s"
I20240502 14:15:50.188957 2901 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:50.211426 2901 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:50.212213 2901 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:50.212487 2901 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:50.213415 2901 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:50.214816 2901 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:50.214915 2901 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:50.215036 2901 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:50.215113 2901 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:50.245818 2901 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:36567
I20240502 14:15:50.245941 3027 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:36567 every 8 connection(s)
I20240502 14:15:50.247359 2901 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:15:50.250664 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 2901
I20240502 14:15:50.250856 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 14:15:50.254382 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.132:0
--local_ip_for_outbound_sockets=127.27.220.132
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:15:50.261257 3028 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:15:50.261610 3028 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:50.262609 3028 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:15:50.264189 2581 ts_manager.cc:194] Registered new tserver with Master: 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:15:50.265002 2581 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:37485
I20240502 14:15:50.320705 2767 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
W20240502 14:15:50.597398 3031 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:50.597729 3031 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:50.597895 3031 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:50.605600 3031 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:50.605734 3031 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:50.605911 3031 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.132
I20240502 14:15:50.615343 3031 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.27.220.132
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.132
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:50.616745 3031 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:50.618655 3031 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:50.630950 3036 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:50.631711 3037 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:50.632265 3039 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:50.633697 3031 server_base.cc:1034] running on GCE node
I20240502 14:15:50.634327 3031 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:50.635273 3031 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:50.636502 3031 hybrid_clock.cc:648] HybridClock initialized: now 1714659350636429 us; error 63 us; skew 500 ppm
I20240502 14:15:50.636827 3031 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:50.643218 3031 webserver.cc:460] Webserver started at http://127.27.220.132:39573/ using document root <none> and password file <none>
I20240502 14:15:50.643986 3031 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:50.644109 3031 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:50.644445 3031 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:50.647380 3031 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
format_stamp: "Formatted at 2024-05-02 14:15:50 on dist-test-slave-j72s"
I20240502 14:15:50.648195 3031 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
format_stamp: "Formatted at 2024-05-02 14:15:50 on dist-test-slave-j72s"
I20240502 14:15:50.653661 3031 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
I20240502 14:15:50.656931 3045 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:50.658243 3031 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:15:50.658411 3031 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
format_stamp: "Formatted at 2024-05-02 14:15:50 on dist-test-slave-j72s"
I20240502 14:15:50.658605 3031 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:50.678552 3031 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:50.679430 3031 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:50.679690 3031 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:50.680620 3031 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:50.682266 3031 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:50.682364 3031 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:50.682490 3031 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:50.682590 3031 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:50.716046 3031 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.132:42265
I20240502 14:15:50.716115 3157 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.132:42265 every 8 connection(s)
I20240502 14:15:50.717401 3031 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 14:15:50.718833 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 3031
I20240502 14:15:50.719005 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 14:15:50.722738 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.133:0
--local_ip_for_outbound_sockets=127.27.220.133
--webserver_interface=127.27.220.133
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:15:50.732420 3158 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:15:50.732781 3158 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:50.733904 3158 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:15:50.735508 2581 ts_manager.cc:194] Registered new tserver with Master: 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:15:50.736433 2581 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.132:36489
I20240502 14:15:50.818885 2898 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
W20240502 14:15:51.075325 3161 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:51.075682 3161 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:51.075852 3161 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:51.083658 3161 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:51.083822 3161 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:51.083997 3161 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.133
I20240502 14:15:51.093523 3161 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.27.220.133
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.133
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:51.094931 3161 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:51.096747 3161 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:51.109573 3166 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:51.110229 3167 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:51.110903 3169 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:51.111755 3161 server_base.cc:1034] running on GCE node
I20240502 14:15:51.112360 3161 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:51.113315 3161 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:51.114508 3161 hybrid_clock.cc:648] HybridClock initialized: now 1714659351114447 us; error 51 us; skew 500 ppm
I20240502 14:15:51.114804 3161 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:51.117117 3161 webserver.cc:460] Webserver started at http://127.27.220.133:45569/ using document root <none> and password file <none>
I20240502 14:15:51.117887 3161 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:51.117985 3161 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:51.118275 3161 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:51.120833 3161 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4"
format_stamp: "Formatted at 2024-05-02 14:15:51 on dist-test-slave-j72s"
I20240502 14:15:51.121603 3161 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4"
format_stamp: "Formatted at 2024-05-02 14:15:51 on dist-test-slave-j72s"
I20240502 14:15:51.126859 3161 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
I20240502 14:15:51.130365 3175 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:51.131690 3161 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 14:15:51.131830 3161 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4"
format_stamp: "Formatted at 2024-05-02 14:15:51 on dist-test-slave-j72s"
I20240502 14:15:51.132053 3161 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:51.146071 3161 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:51.146901 3161 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:51.147132 3161 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:51.148150 3161 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:51.149628 3161 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:51.149713 3161 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:51.149794 3161 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:51.149847 3161 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:51.183066 3161 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.133:41841
I20240502 14:15:51.183180 3287 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.133:41841 every 8 connection(s)
I20240502 14:15:51.184458 3161 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 14:15:51.188596 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 3161
I20240502 14:15:51.188795 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 14:15:51.192657 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.134:0
--local_ip_for_outbound_sockets=127.27.220.134
--webserver_interface=127.27.220.134
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:15:51.198923 3288 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:15:51.199229 3288 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:51.200069 3288 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:15:51.201498 2581 ts_manager.cc:194] Registered new tserver with Master: 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
I20240502 14:15:51.202287 2581 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.133:53021
I20240502 14:15:51.267823 3028 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
W20240502 14:15:51.528507 3291 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:51.528775 3291 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:51.528923 3291 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:51.536252 3291 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:51.536368 3291 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:51.536525 3291 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.134
I20240502 14:15:51.545167 3291 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.134:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.27.220.134
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.134
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:51.546589 3291 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:51.548310 3291 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:51.560308 3296 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:51.561002 3299 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:51.561650 3297 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:51.562356 3291 server_base.cc:1034] running on GCE node
I20240502 14:15:51.562906 3291 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:51.563777 3291 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:51.564987 3291 hybrid_clock.cc:648] HybridClock initialized: now 1714659351564922 us; error 53 us; skew 500 ppm
I20240502 14:15:51.565320 3291 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:51.567541 3291 webserver.cc:460] Webserver started at http://127.27.220.134:44617/ using document root <none> and password file <none>
I20240502 14:15:51.568225 3291 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:51.568336 3291 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:51.568653 3291 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:51.571163 3291 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "763dcf9158e648b496b4faa1a0dc54af"
format_stamp: "Formatted at 2024-05-02 14:15:51 on dist-test-slave-j72s"
I20240502 14:15:51.571864 3291 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "763dcf9158e648b496b4faa1a0dc54af"
format_stamp: "Formatted at 2024-05-02 14:15:51 on dist-test-slave-j72s"
I20240502 14:15:51.576828 3291 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 14:15:51.579815 3305 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:51.580965 3291 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 14:15:51.581104 3291 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "763dcf9158e648b496b4faa1a0dc54af"
format_stamp: "Formatted at 2024-05-02 14:15:51 on dist-test-slave-j72s"
I20240502 14:15:51.581271 3291 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:51.610110 3291 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:51.610939 3291 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:51.611200 3291 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:51.612197 3291 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:51.614043 3291 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:51.614147 3291 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:51.614254 3291 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:51.614317 3291 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:51.644986 3291 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.134:41403
I20240502 14:15:51.645083 3417 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.134:41403 every 8 connection(s)
I20240502 14:15:51.646347 3291 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 14:15:51.655803 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 3291
I20240502 14:15:51.655977 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 14:15:51.658159 3418 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:15:51.658449 3418 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:51.659370 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.135:0
--local_ip_for_outbound_sockets=127.27.220.135
--webserver_interface=127.27.220.135
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:15:51.659399 3418 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:15:51.660764 2581 ts_manager.cc:194] Registered new tserver with Master: 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403)
I20240502 14:15:51.661535 2581 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.134:42121
I20240502 14:15:51.738906 3158 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
W20240502 14:15:51.991250 3421 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:51.991546 3421 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:51.991695 3421 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:51.998905 3421 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:51.999047 3421 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:51.999202 3421 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.135
I20240502 14:15:52.008070 3421 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.135:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.27.220.135
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.135
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:52.009429 3421 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:52.011205 3421 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:52.023262 3426 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:52.024056 3427 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:52.024637 3429 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:52.025605 3421 server_base.cc:1034] running on GCE node
I20240502 14:15:52.026187 3421 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:52.027078 3421 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:52.028276 3421 hybrid_clock.cc:648] HybridClock initialized: now 1714659352028226 us; error 37 us; skew 500 ppm
I20240502 14:15:52.028614 3421 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:52.030869 3421 webserver.cc:460] Webserver started at http://127.27.220.135:46739/ using document root <none> and password file <none>
I20240502 14:15:52.031576 3421 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:52.031695 3421 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:52.032020 3421 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:52.034623 3421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "21e5a60e02fd475bb47e5b3a2f294333"
format_stamp: "Formatted at 2024-05-02 14:15:52 on dist-test-slave-j72s"
I20240502 14:15:52.035377 3421 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "21e5a60e02fd475bb47e5b3a2f294333"
format_stamp: "Formatted at 2024-05-02 14:15:52 on dist-test-slave-j72s"
I20240502 14:15:52.040501 3421 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 14:15:52.043720 3435 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:52.044997 3421 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240502 14:15:52.045142 3421 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "21e5a60e02fd475bb47e5b3a2f294333"
format_stamp: "Formatted at 2024-05-02 14:15:52 on dist-test-slave-j72s"
I20240502 14:15:52.045334 3421 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:52.064515 3421 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:52.065325 3421 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:52.065636 3421 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:52.066540 3421 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:52.067909 3421 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:52.068007 3421 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:52.068121 3421 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:52.068214 3421 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:52.100698 3421 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.135:35331
I20240502 14:15:52.100744 3547 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.135:35331 every 8 connection(s)
I20240502 14:15:52.102015 3421 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 14:15:52.109966 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 3421
I20240502 14:15:52.110136 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 14:15:52.114746 3548 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:15:52.115036 3548 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:52.115831 3548 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:15:52.117180 2581 ts_manager.cc:194] Registered new tserver with Master: 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331)
I20240502 14:15:52.117959 2581 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.135:48045
I20240502 14:15:52.141244 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.136:0
--local_ip_for_outbound_sockets=127.27.220.136
--webserver_interface=127.27.220.136
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:15:52.204706 3288 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
W20240502 14:15:52.479411 3551 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:52.479713 3551 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:15:52.479869 3551 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:52.487409 3551 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:15:52.487535 3551 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:15:52.487695 3551 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.136
I20240502 14:15:52.496351 3551 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.136:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.27.220.136
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.136
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:15:52.497689 3551 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:15:52.499418 3551 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:15:52.511399 3556 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:52.512161 3559 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:15:52.512871 3557 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:15:52.513739 3551 server_base.cc:1034] running on GCE node
I20240502 14:15:52.514310 3551 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:15:52.515192 3551 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:15:52.516382 3551 hybrid_clock.cc:648] HybridClock initialized: now 1714659352516332 us; error 43 us; skew 500 ppm
I20240502 14:15:52.516662 3551 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:15:52.518736 3551 webserver.cc:460] Webserver started at http://127.27.220.136:41141/ using document root <none> and password file <none>
I20240502 14:15:52.519399 3551 fs_manager.cc:359] Metadata directory not provided
I20240502 14:15:52.519505 3551 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:15:52.519783 3551 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:15:52.522120 3551 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "fa04f9da159541bfbbb6ac55db06e07b"
format_stamp: "Formatted at 2024-05-02 14:15:52 on dist-test-slave-j72s"
I20240502 14:15:52.522778 3551 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "fa04f9da159541bfbbb6ac55db06e07b"
format_stamp: "Formatted at 2024-05-02 14:15:52 on dist-test-slave-j72s"
I20240502 14:15:52.527500 3551 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20240502 14:15:52.530404 3565 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:52.531551 3551 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 14:15:52.531694 3551 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "fa04f9da159541bfbbb6ac55db06e07b"
format_stamp: "Formatted at 2024-05-02 14:15:52 on dist-test-slave-j72s"
I20240502 14:15:52.531868 3551 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:15:52.550422 3551 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:15:52.551160 3551 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:15:52.551412 3551 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:15:52.552356 3551 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:15:52.553699 3551 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:15:52.553779 3551 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:52.553855 3551 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:15:52.553910 3551 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:15:52.587857 3551 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.136:39149
I20240502 14:15:52.587922 3677 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.136:39149 every 8 connection(s)
I20240502 14:15:52.589084 3551 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 14:15:52.592303 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 3551
I20240502 14:15:52.592497 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 14:15:52.601930 3678 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:15:52.602209 3678 heartbeater.cc:464] Registering TS with master...
I20240502 14:15:52.603014 3678 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:15:52.604440 2581 ts_manager.cc:194] Registered new tserver with Master: fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149)
I20240502 14:15:52.605234 2581 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.136:57461
I20240502 14:15:52.608400 28530 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240502 14:15:52.624035 28530 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240502 14:15:52.624231 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 3161
I20240502 14:15:52.631772 3283 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:52.664160 3418 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
I20240502 14:15:52.708926 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 3161
I20240502 14:15:52.724777 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 3291
I20240502 14:15:52.732740 3413 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:52.808688 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 3291
I20240502 14:15:52.825174 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 3421
I20240502 14:15:52.832903 3543 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:52.912084 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 3421
I20240502 14:15:52.928635 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 3551
I20240502 14:15:52.936923 3673 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:15:53.017378 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 3551
I20240502 14:15:56.785657 2581 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:15:56.824368 2832 tablet_service.cc:1467] Processing CreateTablet for tablet e9b6901a9a524fd2ab1adb9d730fcd18 (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.826126 2832 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b6901a9a524fd2ab1adb9d730fcd18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.829704 2833 tablet_service.cc:1467] Processing CreateTablet for tablet aa0b526e216f46f0a2fb95fe0be02c46 (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.830291 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa0b526e216f46f0a2fb95fe0be02c46. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.834851 2962 tablet_service.cc:1467] Processing CreateTablet for tablet 8dcd1e1699c8448692fff7c1ed36312a (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.834816 2963 tablet_service.cc:1467] Processing CreateTablet for tablet aa0b526e216f46f0a2fb95fe0be02c46 (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.836455 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa0b526e216f46f0a2fb95fe0be02c46. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.837117 2962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8dcd1e1699c8448692fff7c1ed36312a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.848273 3707 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:15:56.848276 3092 tablet_service.cc:1467] Processing CreateTablet for tablet 8dcd1e1699c8448692fff7c1ed36312a (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.848301 3093 tablet_service.cc:1467] Processing CreateTablet for tablet e9b6901a9a524fd2ab1adb9d730fcd18 (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.849818 3093 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b6901a9a524fd2ab1adb9d730fcd18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.853241 3707 tablet_bootstrap.cc:654] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.854624 3707 log.cc:824] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: Log is configured to *not* fsync() on all Append() calls
I20240502 14:15:56.854799 2702 tablet_service.cc:1467] Processing CreateTablet for tablet aa0b526e216f46f0a2fb95fe0be02c46 (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.855583 3092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8dcd1e1699c8448692fff7c1ed36312a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.856520 2702 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa0b526e216f46f0a2fb95fe0be02c46. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.855576 2700 tablet_service.cc:1467] Processing CreateTablet for tablet 8dcd1e1699c8448692fff7c1ed36312a (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.855034 2701 tablet_service.cc:1467] Processing CreateTablet for tablet e9b6901a9a524fd2ab1adb9d730fcd18 (DEFAULT_TABLE table=rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:56.857640 2701 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b6901a9a524fd2ab1adb9d730fcd18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.858620 3708 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:15:56.858947 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8dcd1e1699c8448692fff7c1ed36312a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:56.863399 3708 tablet_bootstrap.cc:654] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.864778 3708 log.cc:824] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e: Log is configured to *not* fsync() on all Append() calls
I20240502 14:15:56.864990 3707 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:15:56.865399 3707 ts_tablet_manager.cc:1367] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.017s user 0.010s sys 0.005s
I20240502 14:15:56.879416 3707 raft_consensus.cc:348] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.881033 3711 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:15:56.884735 3708 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:15:56.885072 3708 ts_tablet_manager.cc:1367] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.027s user 0.012s sys 0.012s
I20240502 14:15:56.885896 3712 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:15:56.886777 3711 tablet_bootstrap.cc:654] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.886981 3707 raft_consensus.cc:374] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.887120 3707 raft_consensus.cc:729] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.887872 3707 consensus_queue.cc:260] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.888937 3712 tablet_bootstrap.cc:654] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.889663 3711 log.cc:824] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log is configured to *not* fsync() on all Append() calls
I20240502 14:15:56.890003 3712 log.cc:824] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: Log is configured to *not* fsync() on all Append() calls
I20240502 14:15:56.896131 3708 raft_consensus.cc:348] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.896693 3708 raft_consensus.cc:374] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.896821 3708 raft_consensus.cc:729] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.897624 3708 consensus_queue.cc:260] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.899664 3707 ts_tablet_manager.cc:1398] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.034s user 0.014s sys 0.005s
I20240502 14:15:56.900190 3707 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:15:56.901036 3708 ts_tablet_manager.cc:1398] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.016s user 0.012s sys 0.004s
I20240502 14:15:56.901482 3708 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:15:56.902034 3707 tablet_bootstrap.cc:654] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.905313 3712 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:15:56.905472 3711 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:15:56.905678 3712 ts_tablet_manager.cc:1367] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.020s user 0.006s sys 0.010s
I20240502 14:15:56.905802 3711 ts_tablet_manager.cc:1367] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.025s user 0.009s sys 0.004s
I20240502 14:15:56.906525 3708 tablet_bootstrap.cc:654] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.912528 3707 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:15:56.912729 3707 ts_tablet_manager.cc:1367] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.008s
I20240502 14:15:56.912729 3712 raft_consensus.cc:348] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.913221 3712 raft_consensus.cc:374] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.913365 3712 raft_consensus.cc:729] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.913585 3708 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:15:56.913760 3708 ts_tablet_manager.cc:1367] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.000s
I20240502 14:15:56.914135 3707 raft_consensus.cc:348] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.914140 3712 consensus_queue.cc:260] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.914435 3707 raft_consensus.cc:374] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.914621 3707 raft_consensus.cc:729] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.914801 3708 raft_consensus.cc:348] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.915084 3708 raft_consensus.cc:374] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.914954 3707 consensus_queue.cc:260] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.915197 3708 raft_consensus.cc:729] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.915827 3707 ts_tablet_manager.cc:1398] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:56.917395 3712 ts_tablet_manager.cc:1398] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.012s user 0.009s sys 0.001s
I20240502 14:15:56.917251 3711 raft_consensus.cc:348] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.917773 3711 raft_consensus.cc:374] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.917863 3712 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:15:56.917915 3711 raft_consensus.cc:729] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.918597 3711 consensus_queue.cc:260] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.918706 3708 consensus_queue.cc:260] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.920428 3708 ts_tablet_manager.cc:1398] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 14:15:56.921833 3712 tablet_bootstrap.cc:654] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.924044 3711 ts_tablet_manager.cc:1398] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.016s user 0.015s sys 0.000s
I20240502 14:15:56.924324 3712 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:15:56.924463 3712 ts_tablet_manager.cc:1367] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
I20240502 14:15:56.924602 3711 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:15:56.925436 3712 raft_consensus.cc:348] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.925712 3712 raft_consensus.cc:374] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.925845 3712 raft_consensus.cc:729] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.926035 3711 tablet_bootstrap.cc:654] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.926084 3712 consensus_queue.cc:260] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.926577 3712 ts_tablet_manager.cc:1398] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:15:56.926791 3712 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:15:56.928074 3712 tablet_bootstrap.cc:654] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:56.930105 3712 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:15:56.930236 3712 ts_tablet_manager.cc:1367] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:56.930747 3712 raft_consensus.cc:348] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.930920 3712 raft_consensus.cc:374] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.930992 3712 raft_consensus.cc:729] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.931224 3712 consensus_queue.cc:260] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:56.931701 3712 ts_tablet_manager.cc:1398] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 14:15:56.933136 3711 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:15:56.933332 3711 ts_tablet_manager.cc:1367] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.009s user 0.008s sys 0.000s
I20240502 14:15:56.934347 3711 raft_consensus.cc:348] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.934582 3711 raft_consensus.cc:374] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:56.934705 3711 raft_consensus.cc:729] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:15:56.934751 3713 raft_consensus.cc:482] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:56.934945 3711 consensus_queue.cc:260] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.935024 3713 raft_consensus.cc:504] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.935454 3711 ts_tablet_manager.cc:1398] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 14:15:56.936496 3719 raft_consensus.cc:482] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:56.936688 3719 raft_consensus.cc:504] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.937048 3713 leader_election.cc:288] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:15:56.938602 3719 leader_election.cc:288] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:15:56.951224 2853 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:15:56.951531 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:15:56.951786 2853 raft_consensus.cc:2454] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:56.952080 3113 raft_consensus.cc:2454] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 0.
I20240502 14:15:56.952711 2788 leader_election.cc:302] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:56.952704 2658 leader_election.cc:302] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:56.953125 3719 raft_consensus.cc:2781] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:56.953274 3719 raft_consensus.cc:482] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:56.953406 3719 raft_consensus.cc:3037] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:56.953977 3713 raft_consensus.cc:2781] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:56.954128 3713 raft_consensus.cc:482] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:56.954216 3713 raft_consensus.cc:3037] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:56.955971 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:15:56.956447 3719 raft_consensus.cc:504] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.956859 2722 raft_consensus.cc:2379] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c6abc8535fe24d03bf71fb7092072fc4 in current term 1: Already voted for candidate 6b3499bcb84b45a094f7166639704ed4 in this term.
I20240502 14:15:56.957100 3719 leader_election.cc:288] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:15:56.957109 3713 raft_consensus.cc:504] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.957718 2853 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:15:56.957863 3713 leader_election.cc:288] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 election: Requested vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:15:56.958026 2853 raft_consensus.cc:2379] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 6b3499bcb84b45a094f7166639704ed4 in current term 1: Already voted for candidate c6abc8535fe24d03bf71fb7092072fc4 in this term.
I20240502 14:15:56.958586 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:15:56.958727 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:15:56.958773 3113 raft_consensus.cc:3037] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:56.958941 2722 raft_consensus.cc:2379] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate c6abc8535fe24d03bf71fb7092072fc4 in current term 1: Already voted for candidate 6b3499bcb84b45a094f7166639704ed4 in this term.
I20240502 14:15:56.959277 3112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:15:56.959367 3111 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:15:56.960626 2657 leader_election.cc:302] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4; no voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:15:56.960922 3719 raft_consensus.cc:2726] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 14:15:56.961257 3113 raft_consensus.cc:2454] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 1.
I20240502 14:15:56.961620 2788 leader_election.cc:302] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters: 6b3499bcb84b45a094f7166639704ed4
I20240502 14:15:56.961956 3713 raft_consensus.cc:2781] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:56.962370 3713 raft_consensus.cc:686] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Becoming Leader. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Running, Role: LEADER
I20240502 14:15:56.962951 3713 consensus_queue.cc:237] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:56.967665 2579 catalog_manager.cc:5484] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: term changed from 0 to 1, leader changed from <none> to c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130). New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } }
W20240502 14:15:56.973670 3159 tablet.cc:2369] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 14:15:57.003806 3029 tablet.cc:2369] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:15:57.027163 3720 raft_consensus.cc:482] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:57.027402 3720 raft_consensus.cc:504] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:57.029121 3720 leader_election.cc:288] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:15:57.036007 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8dcd1e1699c8448692fff7c1ed36312a" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:15:57.036249 2722 raft_consensus.cc:2454] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:15:57.036115 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8dcd1e1699c8448692fff7c1ed36312a" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:15:57.036707 2983 raft_consensus.cc:2454] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:15:57.036718 3047 leader_election.cc:302] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:15:57.037077 3720 raft_consensus.cc:2781] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:57.037185 3720 raft_consensus.cc:482] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:57.037282 3720 raft_consensus.cc:3037] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:57.039335 3720 raft_consensus.cc:504] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:57.039934 3720 leader_election.cc:288] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:15:57.040408 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8dcd1e1699c8448692fff7c1ed36312a" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:15:57.040484 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8dcd1e1699c8448692fff7c1ed36312a" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:15:57.040632 2983 raft_consensus.cc:3037] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:57.040688 2722 raft_consensus.cc:3037] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:57.043732 2722 raft_consensus.cc:2454] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:15:57.043732 2983 raft_consensus.cc:2454] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:15:57.044178 3046 leader_election.cc:302] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:15:57.044481 3720 raft_consensus.cc:2781] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:57.044884 3720 raft_consensus.cc:686] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Leader. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:15:57.045432 3720 consensus_queue.cc:237] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:57.050137 2579 catalog_manager.cc:5484] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } }
W20240502 14:15:57.053992 2768 tablet.cc:2369] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:15:57.054608 3719 raft_consensus.cc:482] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:57.054771 3719 raft_consensus.cc:504] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:57.055545 3719 leader_election.cc:288] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
W20240502 14:15:57.055631 2899 tablet.cc:2369] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:15:57.056154 2853 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa0b526e216f46f0a2fb95fe0be02c46" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:15:57.056402 2853 raft_consensus.cc:2454] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:57.056895 2658 leader_election.cc:302] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:57.057178 3719 raft_consensus.cc:2781] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:57.057286 3719 raft_consensus.cc:482] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:57.057425 3719 raft_consensus.cc:3037] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:57.059799 3719 raft_consensus.cc:504] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:57.060462 3719 leader_election.cc:288] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:15:57.060973 2853 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa0b526e216f46f0a2fb95fe0be02c46" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:15:57.061139 2853 raft_consensus.cc:3037] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:57.062017 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa0b526e216f46f0a2fb95fe0be02c46" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:15:57.062278 2983 raft_consensus.cc:2454] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:57.062222 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa0b526e216f46f0a2fb95fe0be02c46" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:15:57.062533 2982 raft_consensus.cc:3037] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:57.064180 2853 raft_consensus.cc:2454] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:15:57.064610 2658 leader_election.cc:302] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:57.064924 3719 raft_consensus.cc:2781] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:57.065260 2982 raft_consensus.cc:2454] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:15:57.065555 3719 raft_consensus.cc:686] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:15:57.066084 3719 consensus_queue.cc:237] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:15:57.070394 2579 catalog_manager.cc:5484] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } }
I20240502 14:15:57.397118 3713 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:15:57.404311 3729 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:15:57.447997 3737 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:57.448312 3737 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:15:57.452217 3720 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:15:57.457249 3737 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:15:57.458956 3749 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_0'
I20240502 14:15:57.542348 3719 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:15:57.548458 3735 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 280.964 ms
time per row: 0.140482 ms
I20240502 14:15:57.885221 2580 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:15:57.894436 3093 tablet_service.cc:1467] Processing CreateTablet for tablet 02adec8c6c704326afbedb70578df917 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.894968 3093 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02adec8c6c704326afbedb70578df917. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.894924 2832 tablet_service.cc:1467] Processing CreateTablet for tablet 6cdd66f946db45478a93459ba69702b2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.895385 2832 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cdd66f946db45478a93459ba69702b2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.895866 2962 tablet_service.cc:1467] Processing CreateTablet for tablet 02adec8c6c704326afbedb70578df917 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.896301 2962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02adec8c6c704326afbedb70578df917. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.897992 2702 tablet_service.cc:1467] Processing CreateTablet for tablet 3301dc9fca444af68c8a55dded735b38 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.898435 2702 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3301dc9fca444af68c8a55dded735b38. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.901918 2963 tablet_service.cc:1467] Processing CreateTablet for tablet 6cdd66f946db45478a93459ba69702b2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.902030 3092 tablet_service.cc:1467] Processing CreateTablet for tablet 6cdd66f946db45478a93459ba69702b2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.902384 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cdd66f946db45478a93459ba69702b2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.902506 3092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cdd66f946db45478a93459ba69702b2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.905709 2833 tablet_service.cc:1467] Processing CreateTablet for tablet 02adec8c6c704326afbedb70578df917 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.906199 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02adec8c6c704326afbedb70578df917. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.906420 2961 tablet_service.cc:1467] Processing CreateTablet for tablet 3301dc9fca444af68c8a55dded735b38 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.906895 2961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3301dc9fca444af68c8a55dded735b38. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.907527 3806 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:15:57.910127 2831 tablet_service.cc:1467] Processing CreateTablet for tablet 3301dc9fca444af68c8a55dded735b38 (DEFAULT_TABLE table=rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:57.909497 3806 tablet_bootstrap.cc:654] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.910750 2831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3301dc9fca444af68c8a55dded735b38. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:57.917500 3808 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:15:57.919904 3808 tablet_bootstrap.cc:654] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.920439 3805 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:15:57.922907 3805 tablet_bootstrap.cc:654] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.924314 3806 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:15:57.924546 3806 ts_tablet_manager.cc:1367] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.017s user 0.016s sys 0.000s
I20240502 14:15:57.925824 3806 raft_consensus.cc:348] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:57.926249 3806 raft_consensus.cc:374] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.926422 3806 raft_consensus.cc:729] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.926811 3806 consensus_queue.cc:260] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:57.927563 3806 ts_tablet_manager.cc:1398] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:57.931874 3805 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:15:57.932109 3805 ts_tablet_manager.cc:1367] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.007s
I20240502 14:15:57.932907 3805 raft_consensus.cc:348] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:57.933068 3805 raft_consensus.cc:374] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.933194 3805 raft_consensus.cc:729] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.933430 3805 consensus_queue.cc:260] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:57.934165 3805 ts_tablet_manager.cc:1398] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:15:57.936373 3805 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:15:57.938007 3805 tablet_bootstrap.cc:654] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.938436 3806 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:15:57.940496 3806 tablet_bootstrap.cc:654] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.942656 3811 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:15:57.945052 3811 tablet_bootstrap.cc:654] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.947244 3805 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:15:57.947407 3805 ts_tablet_manager.cc:1367] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.006s
I20240502 14:15:57.947450 3808 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:15:57.947701 3808 ts_tablet_manager.cc:1367] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.030s user 0.017s sys 0.011s
I20240502 14:15:57.948339 3805 raft_consensus.cc:348] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:57.948647 3805 raft_consensus.cc:374] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.948705 3811 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:15:57.948608 3808 raft_consensus.cc:348] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:57.948781 3805 raft_consensus.cc:729] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.948901 3811 ts_tablet_manager.cc:1367] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.006s user 0.005s sys 0.000s
I20240502 14:15:57.948910 3808 raft_consensus.cc:374] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.949074 3808 raft_consensus.cc:729] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.949075 3805 consensus_queue.cc:260] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:57.949445 3808 consensus_queue.cc:260] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:57.949898 3805 ts_tablet_manager.cc:1398] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:15:57.949827 3811 raft_consensus.cc:348] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:57.950109 3811 raft_consensus.cc:374] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.950253 3808 ts_tablet_manager.cc:1398] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:15:57.950250 3811 raft_consensus.cc:729] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.950603 3806 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:15:57.950584 3811 consensus_queue.cc:260] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:57.950831 3806 ts_tablet_manager.cc:1367] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.012s user 0.006s sys 0.000s
I20240502 14:15:57.951301 3811 ts_tablet_manager.cc:1398] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:15:57.951622 3806 raft_consensus.cc:348] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:57.951874 3806 raft_consensus.cc:374] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.951949 3806 raft_consensus.cc:729] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.952232 3806 consensus_queue.cc:260] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:57.952661 3808 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:15:57.952764 3806 ts_tablet_manager.cc:1398] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:15:57.953145 3806 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:15:57.954770 3806 tablet_bootstrap.cc:654] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.956909 3808 tablet_bootstrap.cc:654] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.957023 3806 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:15:57.957195 3806 ts_tablet_manager.cc:1367] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:15:57.958314 3806 raft_consensus.cc:348] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:57.958550 3806 raft_consensus.cc:374] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.958680 3806 raft_consensus.cc:729] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.959000 3808 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:15:57.958987 3806 consensus_queue.cc:260] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:57.959175 3808 ts_tablet_manager.cc:1367] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.002s
I20240502 14:15:57.959650 3806 ts_tablet_manager.cc:1398] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:15:57.960012 3808 raft_consensus.cc:348] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:57.960294 3808 raft_consensus.cc:374] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.960381 3808 raft_consensus.cc:729] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.960629 3808 consensus_queue.cc:260] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:57.961194 3808 ts_tablet_manager.cc:1398] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:15:57.961597 3808 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:15:57.963475 3808 tablet_bootstrap.cc:654] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:57.965523 3808 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:15:57.965694 3808 ts_tablet_manager.cc:1367] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 14:15:57.966334 3808 raft_consensus.cc:348] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:57.966491 3808 raft_consensus.cc:374] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:57.966600 3808 raft_consensus.cc:729] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:15:57.966831 3808 consensus_queue.cc:260] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:57.967283 3808 ts_tablet_manager.cc:1398] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 14:15:57.994026 3733 raft_consensus.cc:482] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:57.994194 3733 raft_consensus.cc:504] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:57.994930 3733 leader_election.cc:288] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:15:57.995568 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6cdd66f946db45478a93459ba69702b2" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:15:57.995878 2982 raft_consensus.cc:2454] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:15:57.996312 3046 leader_election.cc:302] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:15:57.996625 3733 raft_consensus.cc:2781] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:57.996755 3733 raft_consensus.cc:482] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:57.996852 3733 raft_consensus.cc:3037] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:57.999464 3733 raft_consensus.cc:504] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:58.000159 3733 leader_election.cc:288] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:15:58.000692 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6cdd66f946db45478a93459ba69702b2" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:15:58.000908 2982 raft_consensus.cc:3037] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:58.003407 2982 raft_consensus.cc:2454] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:15:58.003520 2853 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6cdd66f946db45478a93459ba69702b2" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:15:58.003850 2853 raft_consensus.cc:2454] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:15:58.003881 3046 leader_election.cc:302] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:15:58.003710 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6cdd66f946db45478a93459ba69702b2" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:15:58.004065 2852 raft_consensus.cc:3037] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:58.004292 3733 raft_consensus.cc:2781] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:58.004576 3733 raft_consensus.cc:686] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Leader. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:15:58.004881 3733 consensus_queue.cc:237] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:58.006371 2852 raft_consensus.cc:2454] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:15:58.007858 2581 catalog_manager.cc:5484] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } }
I20240502 14:15:58.268416 3713 raft_consensus.cc:482] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:58.268590 3713 raft_consensus.cc:504] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:58.269392 3713 leader_election.cc:288] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:15:58.270004 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3301dc9fca444af68c8a55dded735b38" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:15:58.270352 2722 raft_consensus.cc:2454] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 0.
I20240502 14:15:58.270915 2787 leader_election.cc:302] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:58.271237 3713 raft_consensus.cc:2781] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:58.271378 3713 raft_consensus.cc:482] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:58.271467 3713 raft_consensus.cc:3037] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:58.274019 3713 raft_consensus.cc:504] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:58.274701 3713 leader_election.cc:288] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 election: Requested vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:15:58.275130 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3301dc9fca444af68c8a55dded735b38" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:15:58.275353 2722 raft_consensus.cc:3037] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:58.276702 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3301dc9fca444af68c8a55dded735b38" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:15:58.276623 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3301dc9fca444af68c8a55dded735b38" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:15:58.276918 2983 raft_consensus.cc:3037] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:58.277581 2722 raft_consensus.cc:2454] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 1.
I20240502 14:15:58.277940 2787 leader_election.cc:302] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:58.278193 3713 raft_consensus.cc:2781] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:58.278563 3713 raft_consensus.cc:686] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Becoming Leader. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Running, Role: LEADER
I20240502 14:15:58.278879 3713 consensus_queue.cc:237] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:58.279333 2983 raft_consensus.cc:2454] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 1.
I20240502 14:15:58.281450 2581 catalog_manager.cc:5484] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: term changed from 0 to 1, leader changed from <none> to c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130). New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } }
I20240502 14:15:58.333685 3747 raft_consensus.cc:482] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:58.333904 3747 raft_consensus.cc:504] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:58.335438 3747 leader_election.cc:288] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:15:58.342222 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02adec8c6c704326afbedb70578df917" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:15:58.342243 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02adec8c6c704326afbedb70578df917" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:15:58.342445 3113 raft_consensus.cc:2454] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:15:58.342464 2852 raft_consensus.cc:2454] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:15:58.342913 2919 leader_election.cc:302] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:58.343284 3747 raft_consensus.cc:2781] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:58.343420 3747 raft_consensus.cc:482] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:58.343538 3747 raft_consensus.cc:3037] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:58.345718 3747 raft_consensus.cc:504] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:58.346282 3747 leader_election.cc:288] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:15:58.346791 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02adec8c6c704326afbedb70578df917" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:15:58.346766 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "02adec8c6c704326afbedb70578df917" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:15:58.346979 3113 raft_consensus.cc:3037] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:58.346995 2852 raft_consensus.cc:3037] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:58.348816 2852 raft_consensus.cc:2454] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:15:58.348848 3113 raft_consensus.cc:2454] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:15:58.349189 2919 leader_election.cc:302] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:58.349468 3747 raft_consensus.cc:2781] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:58.349875 3747 raft_consensus.cc:686] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Leader. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:15:58.350296 3747 consensus_queue.cc:237] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:58.353864 2581 catalog_manager.cc:5484] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: term changed from 0 to 1, leader changed from <none> to 8cee803d912546e1981521fd8f16bb4e (127.27.220.131). New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } }
I20240502 14:15:58.438658 3733 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:15:58.444136 3733 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:15:58.696399 3713 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:15:58.701745 3713 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:15:58.708264 3829 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:58.708561 3829 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:58.716001 3829 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 14:15:58.873593 3747 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:15:58.925855 2852 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:15:58.927034 3747 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 287.962 ms
time per row: 0.143981 ms
I20240502 14:15:59.163232 2576 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:15:59.171797 2702 tablet_service.cc:1467] Processing CreateTablet for tablet 652af8d083784c078c5b45cca32ece4b (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.172156 2831 tablet_service.cc:1467] Processing CreateTablet for tablet 652af8d083784c078c5b45cca32ece4b (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.172334 2702 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 652af8d083784c078c5b45cca32ece4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.172580 2831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 652af8d083784c078c5b45cca32ece4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.172897 2963 tablet_service.cc:1467] Processing CreateTablet for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.173293 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.177644 3892 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:15:59.178423 3092 tablet_service.cc:1467] Processing CreateTablet for tablet 652af8d083784c078c5b45cca32ece4b (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.178488 3091 tablet_service.cc:1467] Processing CreateTablet for tablet e46c5705a2aa4a3fbb410a284e5ea228 (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.178905 3092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 652af8d083784c078c5b45cca32ece4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.179148 3091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e46c5705a2aa4a3fbb410a284e5ea228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.179508 3892 tablet_bootstrap.cc:654] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.179546 3093 tablet_service.cc:1467] Processing CreateTablet for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.179924 3093 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.179853 2701 tablet_service.cc:1467] Processing CreateTablet for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.180194 2701 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.181852 2700 tablet_service.cc:1467] Processing CreateTablet for tablet e46c5705a2aa4a3fbb410a284e5ea228 (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.182243 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e46c5705a2aa4a3fbb410a284e5ea228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.183648 2833 tablet_service.cc:1467] Processing CreateTablet for tablet e46c5705a2aa4a3fbb410a284e5ea228 (DEFAULT_TABLE table=rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:15:59.184069 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e46c5705a2aa4a3fbb410a284e5ea228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:15:59.190191 3895 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:15:59.191699 3893 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:15:59.192159 3895 tablet_bootstrap.cc:654] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.193693 3893 tablet_bootstrap.cc:654] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.196264 3892 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:15:59.196532 3892 ts_tablet_manager.cc:1367] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.019s user 0.003s sys 0.015s
I20240502 14:15:59.197697 3892 raft_consensus.cc:348] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.198089 3892 raft_consensus.cc:374] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.198271 3892 raft_consensus.cc:729] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.198629 3892 consensus_queue.cc:260] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.199585 3892 ts_tablet_manager.cc:1398] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:15:59.203972 3893 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:15:59.204205 3893 ts_tablet_manager.cc:1367] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.004s
I20240502 14:15:59.205448 3893 raft_consensus.cc:348] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.205864 3893 raft_consensus.cc:374] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.206054 3893 raft_consensus.cc:729] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.206426 3893 consensus_queue.cc:260] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.207406 3893 ts_tablet_manager.cc:1398] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:59.208700 3898 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:15:59.209539 3893 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:15:59.210031 3895 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:15:59.210237 3895 ts_tablet_manager.cc:1367] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.020s user 0.011s sys 0.006s
I20240502 14:15:59.210384 3898 tablet_bootstrap.cc:654] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.211025 3893 tablet_bootstrap.cc:654] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.211115 3895 raft_consensus.cc:348] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.211376 3895 raft_consensus.cc:374] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.211597 3895 raft_consensus.cc:729] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.211884 3895 consensus_queue.cc:260] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.212648 3895 ts_tablet_manager.cc:1398] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:15:59.212915 3893 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:15:59.212935 3895 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:15:59.213135 3893 ts_tablet_manager.cc:1367] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:15:59.213212 3898 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:15:59.213410 3898 ts_tablet_manager.cc:1367] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
I20240502 14:15:59.213910 3893 raft_consensus.cc:348] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.214207 3893 raft_consensus.cc:374] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.214229 3898 raft_consensus.cc:348] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.214335 3893 raft_consensus.cc:729] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.214581 3893 consensus_queue.cc:260] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.215044 3893 ts_tablet_manager.cc:1398] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:15:59.214498 3898 raft_consensus.cc:374] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.215253 3898 raft_consensus.cc:729] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.215691 3898 consensus_queue.cc:260] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.216454 3898 ts_tablet_manager.cc:1398] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:15:59.216799 3898 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:15:59.215821 3895 tablet_bootstrap.cc:654] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.218533 3898 tablet_bootstrap.cc:654] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.220476 3898 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:15:59.220541 3895 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:15:59.220655 3898 ts_tablet_manager.cc:1367] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.004s user 0.001s sys 0.003s
I20240502 14:15:59.220758 3895 ts_tablet_manager.cc:1367] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.001s
I20240502 14:15:59.221194 3898 raft_consensus.cc:348] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.221350 3898 raft_consensus.cc:374] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.221418 3898 raft_consensus.cc:729] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.221462 3895 raft_consensus.cc:348] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.221650 3895 raft_consensus.cc:374] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.221621 3898 consensus_queue.cc:260] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.221697 3895 raft_consensus.cc:729] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.221859 3895 consensus_queue.cc:260] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.221975 3898 ts_tablet_manager.cc:1398] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 14:15:59.222241 3898 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:15:59.222313 3895 ts_tablet_manager.cc:1398] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 14:15:59.222559 3895 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:15:59.223709 3898 tablet_bootstrap.cc:654] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.224025 3895 tablet_bootstrap.cc:654] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:15:59.226008 3895 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:15:59.226197 3895 ts_tablet_manager.cc:1367] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:15:59.226327 3898 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:15:59.226464 3898 ts_tablet_manager.cc:1367] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
I20240502 14:15:59.226907 3895 raft_consensus.cc:348] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.227022 3895 raft_consensus.cc:374] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.227072 3895 raft_consensus.cc:729] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.227192 3895 consensus_queue.cc:260] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.227233 3898 raft_consensus.cc:348] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.227406 3898 raft_consensus.cc:374] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:15:59.227519 3898 raft_consensus.cc:729] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:15:59.227659 3895 ts_tablet_manager.cc:1398] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 14:15:59.227725 3898 consensus_queue.cc:260] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.228176 3898 ts_tablet_manager.cc:1398] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:15:59.254884 3719 raft_consensus.cc:482] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:59.255045 3719 raft_consensus.cc:504] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.255715 3719 leader_election.cc:288] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:15:59.256162 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:15:59.256254 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:15:59.256408 3113 raft_consensus.cc:2454] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:59.256484 2983 raft_consensus.cc:2454] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:59.256793 2657 leader_election.cc:302] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:15:59.257087 3719 raft_consensus.cc:2781] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:59.257184 3719 raft_consensus.cc:482] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:59.257272 3719 raft_consensus.cc:3037] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.259526 3719 raft_consensus.cc:504] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.260094 3719 leader_election.cc:288] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:15:59.260470 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:15:59.260648 3113 raft_consensus.cc:3037] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.260596 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:15:59.260835 2983 raft_consensus.cc:3037] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.262558 3113 raft_consensus.cc:2454] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:15:59.262917 2657 leader_election.cc:302] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:15:59.263145 3719 raft_consensus.cc:2781] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:59.263731 3719 raft_consensus.cc:686] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:15:59.263875 2983 raft_consensus.cc:2454] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:15:59.264083 3719 consensus_queue.cc:237] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:15:59.266706 2576 catalog_manager.cc:5484] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } }
I20240502 14:15:59.518028 3719 raft_consensus.cc:482] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:59.518203 3719 raft_consensus.cc:504] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.518842 3719 leader_election.cc:288] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:15:59.519203 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "652af8d083784c078c5b45cca32ece4b" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:15:59.519341 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "652af8d083784c078c5b45cca32ece4b" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:15:59.519430 2851 raft_consensus.cc:2454] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:59.519580 3113 raft_consensus.cc:2454] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:59.519815 2658 leader_election.cc:302] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:59.520049 3719 raft_consensus.cc:2781] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:59.520166 3719 raft_consensus.cc:482] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:59.520251 3719 raft_consensus.cc:3037] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.522226 3719 raft_consensus.cc:504] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.522744 3719 leader_election.cc:288] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:15:59.523119 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "652af8d083784c078c5b45cca32ece4b" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:15:59.523241 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "652af8d083784c078c5b45cca32ece4b" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:15:59.523344 2851 raft_consensus.cc:3037] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.523461 3113 raft_consensus.cc:3037] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.526163 2851 raft_consensus.cc:2454] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:15:59.526161 3113 raft_consensus.cc:2454] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:15:59.526281 3719 raft_consensus.cc:482] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:15:59.526549 2658 leader_election.cc:302] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:59.526470 3719 raft_consensus.cc:504] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.526899 3905 raft_consensus.cc:2781] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:59.527395 3719 leader_election.cc:288] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:15:59.527405 3905 raft_consensus.cc:686] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:15:59.527701 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:15:59.527760 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:15:59.527911 2851 raft_consensus.cc:2454] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:59.527947 3113 raft_consensus.cc:2454] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:15:59.528117 3905 consensus_queue.cc:237] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:15:59.528295 2658 leader_election.cc:302] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:15:59.528622 3907 raft_consensus.cc:2781] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:15:59.528719 3907 raft_consensus.cc:482] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:15:59.528838 3907 raft_consensus.cc:3037] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.530931 3907 raft_consensus.cc:504] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.531584 3907 leader_election.cc:288] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:15:59.531936 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:15:59.532058 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:15:59.532114 3113 raft_consensus.cc:3037] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.532267 2851 raft_consensus.cc:3037] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:15:59.533987 3113 raft_consensus.cc:2454] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:15:59.534008 2576 catalog_manager.cc:5484] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } }
I20240502 14:15:59.534906 2851 raft_consensus.cc:2454] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:15:59.535301 2657 leader_election.cc:302] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:15:59.535535 3907 raft_consensus.cc:2781] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:15:59.535672 3907 raft_consensus.cc:686] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:15:59.536268 3907 consensus_queue.cc:237] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:15:59.539511 2576 catalog_manager.cc:5484] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } }
I20240502 14:15:59.690343 3905 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:15:59.696746 3905 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:15:59.889986 3910 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:15:59.890262 3910 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:15:59.897295 3910 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240502 14:16:00.028033 3113 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:16:00.028429 2851 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:16:00.029420 3719 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:00.030061 3719 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:00.043262 2852 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:16:00.044888 3111 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:16:00.045439 3907 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:00.045869 3719 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:00.102825 2658 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:44843, user_credentials={real_user=slave}} blocked reactor thread for 57267.6us
I20240502 14:16:00.164584 3913 mvcc.cc:205] Tried to move back new op lower bound from 7023244738716782592 to 7023244736662949888. Current Snapshot: MvccSnapshot[applied={T|T < 7023244738716782592}]
INSERT report
rows total: 2000
time total: 263.098 ms
time per row: 0.131549 ms
I20240502 14:16:00.297320 2581 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:00.306782 2700 tablet_service.cc:1467] Processing CreateTablet for tablet e4f8998b30c34530a308f78e61138315 (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.306777 2833 tablet_service.cc:1467] Processing CreateTablet for tablet e4f8998b30c34530a308f78e61138315 (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.307243 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e4f8998b30c34530a308f78e61138315. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.307252 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e4f8998b30c34530a308f78e61138315. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.308301 2963 tablet_service.cc:1467] Processing CreateTablet for tablet e4f8998b30c34530a308f78e61138315 (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.308660 2961 tablet_service.cc:1467] Processing CreateTablet for tablet 9fd6780b78f849198bc91ed9ae2d08be (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.309221 3092 tablet_service.cc:1467] Processing CreateTablet for tablet 03ebc1eccb85405a8f5ea7582d8822f9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.309701 3092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 03ebc1eccb85405a8f5ea7582d8822f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.309959 2702 tablet_service.cc:1467] Processing CreateTablet for tablet 03ebc1eccb85405a8f5ea7582d8822f9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.310292 2702 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 03ebc1eccb85405a8f5ea7582d8822f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.310241 3091 tablet_service.cc:1467] Processing CreateTablet for tablet 9fd6780b78f849198bc91ed9ae2d08be (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.310612 3091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9fd6780b78f849198bc91ed9ae2d08be. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.308346 2962 tablet_service.cc:1467] Processing CreateTablet for tablet 03ebc1eccb85405a8f5ea7582d8822f9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.308771 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e4f8998b30c34530a308f78e61138315. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.314188 2831 tablet_service.cc:1467] Processing CreateTablet for tablet 9fd6780b78f849198bc91ed9ae2d08be (DEFAULT_TABLE table=rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:00.314695 2831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9fd6780b78f849198bc91ed9ae2d08be. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.317669 2962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 03ebc1eccb85405a8f5ea7582d8822f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.320048 2961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9fd6780b78f849198bc91ed9ae2d08be. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:00.325024 3969 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:00.327436 3969 tablet_bootstrap.cc:654] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:00.333456 3970 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:00.335636 3969 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:00.335907 3969 ts_tablet_manager.cc:1367] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.011s user 0.009s sys 0.000s
I20240502 14:16:00.336313 3970 tablet_bootstrap.cc:654] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:00.336932 3969 raft_consensus.cc:348] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.337249 3969 raft_consensus.cc:374] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.337406 3969 raft_consensus.cc:729] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.337728 3969 consensus_queue.cc:260] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.338157 3968 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:00.339032 3969 ts_tablet_manager.cc:1398] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:00.339648 3969 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:00.339673 3968 tablet_bootstrap.cc:654] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:00.340726 3720 raft_consensus.cc:482] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:00.340881 3720 raft_consensus.cc:504] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.341710 3720 leader_election.cc:288] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:00.342151 3969 tablet_bootstrap.cc:654] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:00.342547 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03ebc1eccb85405a8f5ea7582d8822f9" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:00.343590 3973 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
W20240502 14:16:00.343760 3047 leader_election.cc:341] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:16:00.344496 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03ebc1eccb85405a8f5ea7582d8822f9" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:00.345353 3973 tablet_bootstrap.cc:654] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
W20240502 14:16:00.345803 3046 leader_election.cc:341] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:16:00.345975 3046 leader_election.cc:302] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2; no voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e
I20240502 14:16:00.347249 3720 raft_consensus.cc:2726] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:16:00.347563 3968 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:00.347761 3968 ts_tablet_manager.cc:1367] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.003s
I20240502 14:16:00.348663 3970 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:00.348660 3968 raft_consensus.cc:348] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.348974 3968 raft_consensus.cc:374] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.348860 3970 ts_tablet_manager.cc:1367] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.016s user 0.000s sys 0.008s
I20240502 14:16:00.349167 3968 raft_consensus.cc:729] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.349436 3968 consensus_queue.cc:260] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.349915 3970 raft_consensus.cc:348] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.350021 3968 ts_tablet_manager.cc:1398] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:00.350108 3970 raft_consensus.cc:374] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.350186 3970 raft_consensus.cc:729] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.350294 3968 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:00.350394 3970 consensus_queue.cc:260] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.351455 3970 ts_tablet_manager.cc:1398] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:00.351745 3970 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:00.352011 3968 tablet_bootstrap.cc:654] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:00.353410 3970 tablet_bootstrap.cc:654] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:00.356119 3969 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:00.356359 3969 ts_tablet_manager.cc:1367] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.017s user 0.000s sys 0.004s
I20240502 14:16:00.357342 3969 raft_consensus.cc:348] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.357633 3969 raft_consensus.cc:374] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.357726 3969 raft_consensus.cc:729] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.358769 3969 consensus_queue.cc:260] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.359041 3970 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:00.359619 3969 ts_tablet_manager.cc:1398] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.003s user 0.002s sys 0.001s
I20240502 14:16:00.359807 3970 ts_tablet_manager.cc:1367] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.002s
I20240502 14:16:00.360617 3970 raft_consensus.cc:348] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.360850 3970 raft_consensus.cc:374] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.360958 3970 raft_consensus.cc:729] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.361388 3968 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:00.361579 3968 ts_tablet_manager.cc:1367] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.008s
I20240502 14:16:00.361212 3970 consensus_queue.cc:260] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.362836 3968 raft_consensus.cc:348] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.363065 3968 raft_consensus.cc:374] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.363144 3968 raft_consensus.cc:729] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.363427 3968 consensus_queue.cc:260] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.363719 3720 raft_consensus.cc:482] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:00.363868 3720 raft_consensus.cc:504] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.364137 3968 ts_tablet_manager.cc:1398] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:00.364605 3720 leader_election.cc:288] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:00.365615 3970 ts_tablet_manager.cc:1398] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.006s user 0.001s sys 0.001s
I20240502 14:16:00.365541 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9fd6780b78f849198bc91ed9ae2d08be" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:00.365646 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9fd6780b78f849198bc91ed9ae2d08be" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:00.365891 2851 raft_consensus.cc:2454] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
W20240502 14:16:00.366101 3046 leader_election.cc:341] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:16:00.366425 3049 leader_election.cc:302] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters: 8cee803d912546e1981521fd8f16bb4e
I20240502 14:16:00.367019 3973 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:00.367211 3973 ts_tablet_manager.cc:1367] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.024s user 0.005s sys 0.000s
I20240502 14:16:00.367460 3720 raft_consensus.cc:2781] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:00.367583 3720 raft_consensus.cc:482] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:00.367673 3720 raft_consensus.cc:3037] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.369112 3973 raft_consensus.cc:348] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.369336 3973 raft_consensus.cc:374] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.369452 3973 raft_consensus.cc:729] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.369503 3720 raft_consensus.cc:504] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.369711 3973 consensus_queue.cc:260] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.370086 3720 leader_election.cc:288] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:00.370425 3973 ts_tablet_manager.cc:1398] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 14:16:00.370447 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9fd6780b78f849198bc91ed9ae2d08be" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:00.370791 3973 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:00.371079 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9fd6780b78f849198bc91ed9ae2d08be" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:00.371296 2851 raft_consensus.cc:3037] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.373311 2851 raft_consensus.cc:2454] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:00.374505 3973 tablet_bootstrap.cc:654] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
W20240502 14:16:00.371057 3046 leader_election.cc:341] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:16:00.375370 3049 leader_election.cc:302] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters: 8cee803d912546e1981521fd8f16bb4e
I20240502 14:16:00.375723 3720 raft_consensus.cc:2781] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:00.376574 3973 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:00.376726 3973 ts_tablet_manager.cc:1367] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.003s
I20240502 14:16:00.377435 3973 raft_consensus.cc:348] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.377804 3973 raft_consensus.cc:374] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.377928 3973 raft_consensus.cc:729] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.378211 3973 consensus_queue.cc:260] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.378890 3973 ts_tablet_manager.cc:1398] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:00.379218 3973 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:00.379261 3720 raft_consensus.cc:686] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Leader. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:16:00.379573 3720 consensus_queue.cc:237] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.380941 3973 tablet_bootstrap.cc:654] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:00.382339 2581 catalog_manager.cc:5484] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:00.386533 3973 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:00.386672 3973 ts_tablet_manager.cc:1367] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.008s user 0.004s sys 0.000s
I20240502 14:16:00.387579 3973 raft_consensus.cc:348] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.387820 3973 raft_consensus.cc:374] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:00.387931 3973 raft_consensus.cc:729] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:00.388162 3973 consensus_queue.cc:260] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.388756 3973 ts_tablet_manager.cc:1398] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:00.492242 3946 raft_consensus.cc:482] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:00.492460 3946 raft_consensus.cc:504] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.493165 3946 leader_election.cc:288] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:00.493717 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03ebc1eccb85405a8f5ea7582d8822f9" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:00.493736 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03ebc1eccb85405a8f5ea7582d8822f9" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:00.493954 2982 raft_consensus.cc:2454] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:00.493961 3113 raft_consensus.cc:2454] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:00.494341 2655 leader_election.cc:302] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:00.494594 3946 raft_consensus.cc:2781] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:00.494690 3946 raft_consensus.cc:482] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:00.494752 3946 raft_consensus.cc:3037] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.496824 3946 raft_consensus.cc:504] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.497345 3946 leader_election.cc:288] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:00.497743 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03ebc1eccb85405a8f5ea7582d8822f9" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:00.497824 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "03ebc1eccb85405a8f5ea7582d8822f9" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:00.497942 3113 raft_consensus.cc:3037] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.497975 2982 raft_consensus.cc:3037] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.499732 2982 raft_consensus.cc:2454] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:00.499734 3113 raft_consensus.cc:2454] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:00.500139 2655 leader_election.cc:302] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:00.500397 3946 raft_consensus.cc:2781] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:00.500535 3946 raft_consensus.cc:686] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:00.500809 3946 consensus_queue.cc:237] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:00.503521 2580 catalog_manager.cc:5484] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:00.575233 3747 raft_consensus.cc:482] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:00.575484 3747 raft_consensus.cc:504] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:00.576216 3747 leader_election.cc:288] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:00.576822 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9fd6780b78f849198bc91ed9ae2d08be" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:00.576854 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9fd6780b78f849198bc91ed9ae2d08be" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:00.577121 2851 raft_consensus.cc:2379] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 8cee803d912546e1981521fd8f16bb4e in current term 1: Already voted for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in this term.
I20240502 14:16:00.577576 2919 leader_election.cc:302] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e; no voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:16:00.577838 3747 raft_consensus.cc:3037] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.579702 3747 raft_consensus.cc:2726] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:16:00.691357 3877 raft_consensus.cc:482] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:00.691565 3877 raft_consensus.cc:504] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.692267 3877 leader_election.cc:288] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:00.692900 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e4f8998b30c34530a308f78e61138315" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:00.692936 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e4f8998b30c34530a308f78e61138315" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:00.693156 2722 raft_consensus.cc:2454] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 0.
I20240502 14:16:00.693183 2982 raft_consensus.cc:2454] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 0.
I20240502 14:16:00.693655 2787 leader_election.cc:302] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:00.693996 3877 raft_consensus.cc:2781] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:00.694100 3877 raft_consensus.cc:482] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:00.694208 3877 raft_consensus.cc:3037] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.696424 3877 raft_consensus.cc:504] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.697000 3877 leader_election.cc:288] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 election: Requested vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:00.697485 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e4f8998b30c34530a308f78e61138315" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:00.697571 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e4f8998b30c34530a308f78e61138315" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:00.697785 2722 raft_consensus.cc:3037] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.697821 2982 raft_consensus.cc:3037] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:00.700630 2722 raft_consensus.cc:2454] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 1.
I20240502 14:16:00.700630 2982 raft_consensus.cc:2454] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 1.
I20240502 14:16:00.701138 2787 leader_election.cc:302] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:00.701504 3877 raft_consensus.cc:2781] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:00.701902 3877 raft_consensus.cc:686] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Becoming Leader. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Running, Role: LEADER
I20240502 14:16:00.702203 3877 consensus_queue.cc:237] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:00.705299 2580 catalog_manager.cc:5484] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: term changed from 0 to 1, leader changed from <none> to c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130). New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:00.893427 3720 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:00.893956 3720 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:00.940336 3946 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:00.946828 3946 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:01.085139 3877 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:01.090494 3984 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:01.155683 3986 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:01.155987 3986 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:01.163268 3986 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
INSERT report
rows total: 2000
time total: 285.853 ms
time per row: 0.142926 ms
I20240502 14:16:01.587386 2581 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:01.596000 2833 tablet_service.cc:1467] Processing CreateTablet for tablet a975b36b40b74545a122ff11a6e4bee1 (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.596557 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a975b36b40b74545a122ff11a6e4bee1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.596519 2962 tablet_service.cc:1467] Processing CreateTablet for tablet a975b36b40b74545a122ff11a6e4bee1 (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.597035 2962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a975b36b40b74545a122ff11a6e4bee1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.597364 2831 tablet_service.cc:1467] Processing CreateTablet for tablet 88148a10290c4c48b87ab7e6aae5694c (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.597800 2831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 88148a10290c4c48b87ab7e6aae5694c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.601388 4052 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:01.601668 2700 tablet_service.cc:1467] Processing CreateTablet for tablet a975b36b40b74545a122ff11a6e4bee1 (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.602092 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a975b36b40b74545a122ff11a6e4bee1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.602195 2961 tablet_service.cc:1467] Processing CreateTablet for tablet 4f3030ae5dc2425586db21b6667e2deb (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.602576 2961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f3030ae5dc2425586db21b6667e2deb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.603380 4052 tablet_bootstrap.cc:654] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.603896 3091 tablet_service.cc:1467] Processing CreateTablet for tablet 88148a10290c4c48b87ab7e6aae5694c (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.604352 3091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 88148a10290c4c48b87ab7e6aae5694c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.605979 3092 tablet_service.cc:1467] Processing CreateTablet for tablet 4f3030ae5dc2425586db21b6667e2deb (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.605983 2702 tablet_service.cc:1467] Processing CreateTablet for tablet 88148a10290c4c48b87ab7e6aae5694c (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.606400 3092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f3030ae5dc2425586db21b6667e2deb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.606444 2702 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 88148a10290c4c48b87ab7e6aae5694c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.611299 2701 tablet_service.cc:1467] Processing CreateTablet for tablet 4f3030ae5dc2425586db21b6667e2deb (DEFAULT_TABLE table=rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:01.611733 2701 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f3030ae5dc2425586db21b6667e2deb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:01.616504 4053 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:01.618409 4053 tablet_bootstrap.cc:654] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.621464 4055 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:01.622956 4055 tablet_bootstrap.cc:654] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.624845 4058 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:01.627079 4055 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:01.627652 4058 tablet_bootstrap.cc:654] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.627858 4055 ts_tablet_manager.cc:1367] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
I20240502 14:16:01.629071 4055 raft_consensus.cc:348] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.630195 4055 raft_consensus.cc:374] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.630358 4055 raft_consensus.cc:729] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.630594 4058 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:01.630657 4052 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:01.630720 4058 ts_tablet_manager.cc:1367] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.002s
I20240502 14:16:01.630873 4052 ts_tablet_manager.cc:1367] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.030s user 0.010s sys 0.019s
I20240502 14:16:01.631143 4053 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:01.631494 4058 raft_consensus.cc:348] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.631443 4055 consensus_queue.cc:260] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.631784 4058 raft_consensus.cc:374] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.631767 4052 raft_consensus.cc:348] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.631933 4058 raft_consensus.cc:729] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.632033 4052 raft_consensus.cc:374] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.632200 4052 raft_consensus.cc:729] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.632310 4055 ts_tablet_manager.cc:1398] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:16:01.632328 4058 consensus_queue.cc:260] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.632550 4053 ts_tablet_manager.cc:1367] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.016s user 0.003s sys 0.004s
I20240502 14:16:01.633080 4058 ts_tablet_manager.cc:1398] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:16:01.632496 4052 consensus_queue.cc:260] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.633499 4053 raft_consensus.cc:348] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.633827 4058 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:01.634697 4052 ts_tablet_manager.cc:1398] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:16:01.635058 4052 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:01.636020 4055 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:01.638059 4053 raft_consensus.cc:374] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.638268 4053 raft_consensus.cc:729] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.638823 4052 tablet_bootstrap.cc:654] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.638664 4053 consensus_queue.cc:260] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.638900 4058 tablet_bootstrap.cc:654] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.640964 4055 tablet_bootstrap.cc:654] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.641284 4053 ts_tablet_manager.cc:1398] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240502 14:16:01.642994 4053 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:01.645458 4053 tablet_bootstrap.cc:654] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.645709 4058 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:01.646088 4058 ts_tablet_manager.cc:1367] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.013s user 0.005s sys 0.001s
I20240502 14:16:01.647257 4058 raft_consensus.cc:348] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.648072 4058 raft_consensus.cc:374] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.648252 4055 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:01.648226 4058 raft_consensus.cc:729] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.648388 4052 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:01.648533 4055 ts_tablet_manager.cc:1367] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.005s
I20240502 14:16:01.648635 4052 ts_tablet_manager.cc:1367] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.014s user 0.008s sys 0.002s
I20240502 14:16:01.648569 4058 consensus_queue.cc:260] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.649376 4055 raft_consensus.cc:348] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.649575 4058 ts_tablet_manager.cc:1398] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:16:01.649684 4055 raft_consensus.cc:374] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.649827 4055 raft_consensus.cc:729] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.650055 4058 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:01.650177 4055 consensus_queue.cc:260] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.651088 4052 raft_consensus.cc:348] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.651386 4052 raft_consensus.cc:374] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.651497 4052 raft_consensus.cc:729] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.652077 4058 tablet_bootstrap.cc:654] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:01.652570 4052 consensus_queue.cc:260] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.653156 4052 ts_tablet_manager.cc:1398] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240502 14:16:01.654361 4055 ts_tablet_manager.cc:1398] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 14:16:01.656325 4053 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:01.656554 4053 ts_tablet_manager.cc:1367] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.005s
I20240502 14:16:01.657433 4053 raft_consensus.cc:348] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.657747 4053 raft_consensus.cc:374] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.657846 4053 raft_consensus.cc:729] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.658064 4058 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:01.658138 4053 consensus_queue.cc:260] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.658308 4058 ts_tablet_manager.cc:1367] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.008s user 0.004s sys 0.004s
I20240502 14:16:01.658497 3987 raft_consensus.cc:482] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:01.658646 3987 raft_consensus.cc:504] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.659086 4058 raft_consensus.cc:348] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.659274 3987 leader_election.cc:288] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:01.659549 4053 ts_tablet_manager.cc:1398] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:16:01.659663 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "88148a10290c4c48b87ab7e6aae5694c" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:01.659790 4058 raft_consensus.cc:374] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:01.659850 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "88148a10290c4c48b87ab7e6aae5694c" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:01.659912 2722 raft_consensus.cc:2454] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:01.659916 4058 raft_consensus.cc:729] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:01.660037 2851 raft_consensus.cc:2454] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:01.660459 3047 leader_election.cc:302] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:01.661104 3987 raft_consensus.cc:2781] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:01.661031 4058 consensus_queue.cc:260] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.661216 3987 raft_consensus.cc:482] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:01.661311 3987 raft_consensus.cc:3037] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:01.661620 4058 ts_tablet_manager.cc:1398] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 14:16:01.662182 3720 raft_consensus.cc:482] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:01.662343 3720 raft_consensus.cc:504] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.662968 3720 leader_election.cc:288] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:01.663491 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:01.663600 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:01.663736 2722 raft_consensus.cc:2454] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:01.663815 2983 raft_consensus.cc:2454] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:01.664173 3047 leader_election.cc:302] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:01.664491 3720 raft_consensus.cc:2781] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:01.664584 3720 raft_consensus.cc:482] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:01.664119 3987 raft_consensus.cc:504] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.664711 3720 raft_consensus.cc:3037] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:01.665686 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "88148a10290c4c48b87ab7e6aae5694c" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:01.665925 2722 raft_consensus.cc:3037] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:01.666642 3987 leader_election.cc:288] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Requested vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:01.667173 3720 raft_consensus.cc:504] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.667186 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "88148a10290c4c48b87ab7e6aae5694c" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:01.667394 2851 raft_consensus.cc:3037] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:01.668085 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:01.668076 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:01.668289 2721 raft_consensus.cc:3037] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:01.668303 2983 raft_consensus.cc:3037] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:01.668833 2722 raft_consensus.cc:2454] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:01.669802 3047 leader_election.cc:302] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:01.670070 2851 raft_consensus.cc:2454] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:01.671145 2983 raft_consensus.cc:2454] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:01.671514 2721 raft_consensus.cc:2454] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:01.671556 3046 leader_election.cc:302] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:01.671940 3720 leader_election.cc:288] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:01.672268 3720 raft_consensus.cc:2781] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:01.672273 4065 raft_consensus.cc:2781] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:01.672398 3720 raft_consensus.cc:686] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Leader. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:16:01.672621 3720 consensus_queue.cc:237] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.673290 4065 raft_consensus.cc:686] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Leader. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:16:01.673779 4065 consensus_queue.cc:237] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:01.675958 2572 catalog_manager.cc:5484] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:01.680454 2572 catalog_manager.cc:5484] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:01.851001 3747 raft_consensus.cc:482] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:01.851158 3747 raft_consensus.cc:504] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.851900 3747 leader_election.cc:288] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:01.852391 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a975b36b40b74545a122ff11a6e4bee1" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:01.852684 2851 raft_consensus.cc:2454] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:01.853150 2919 leader_election.cc:302] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:01.853478 3747 raft_consensus.cc:2781] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:01.853637 3747 raft_consensus.cc:482] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:01.853754 3747 raft_consensus.cc:3037] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:01.856428 3747 raft_consensus.cc:504] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.857024 3747 leader_election.cc:288] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:01.857479 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a975b36b40b74545a122ff11a6e4bee1" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:01.857714 2851 raft_consensus.cc:3037] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:01.859755 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a975b36b40b74545a122ff11a6e4bee1" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:01.859874 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a975b36b40b74545a122ff11a6e4bee1" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:01.860028 2721 raft_consensus.cc:2454] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:01.860402 2851 raft_consensus.cc:2454] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:01.860764 2919 leader_election.cc:302] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters: 6b3499bcb84b45a094f7166639704ed4
I20240502 14:16:01.861054 3747 raft_consensus.cc:2781] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:01.861407 3747 raft_consensus.cc:686] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Leader. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:01.861673 3747 consensus_queue.cc:237] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:01.864809 2572 catalog_manager.cc:5484] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: term changed from 0 to 1, leader changed from <none> to 8cee803d912546e1981521fd8f16bb4e (127.27.220.131). New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:02.099558 4065 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:02.104610 4065 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:02.166057 4065 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:02.170634 3987 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:02.260495 4073 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:02.260775 4073 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:02.268239 4073 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:16:02.286655 3747 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:02.290426 2722 raft_consensus.cc:3037] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.294080 4093 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_4'
INSERT report
rows total: 2000
time total: 237.289 ms
time per row: 0.118644 ms
I20240502 14:16:02.667913 2577 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:02.677024 2833 tablet_service.cc:1467] Processing CreateTablet for tablet 7f7589fa40dd4af09cc87e6e793bcd8a (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.677213 3091 tablet_service.cc:1467] Processing CreateTablet for tablet 7f7589fa40dd4af09cc87e6e793bcd8a (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.677515 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7f7589fa40dd4af09cc87e6e793bcd8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.677680 3091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7f7589fa40dd4af09cc87e6e793bcd8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.678105 2831 tablet_service.cc:1467] Processing CreateTablet for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.678284 2962 tablet_service.cc:1467] Processing CreateTablet for tablet 7f7589fa40dd4af09cc87e6e793bcd8a (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.678432 2831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.678709 2962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7f7589fa40dd4af09cc87e6e793bcd8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.681886 2961 tablet_service.cc:1467] Processing CreateTablet for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.682240 2961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.686511 3092 tablet_service.cc:1467] Processing CreateTablet for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.686889 3092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.686852 2963 tablet_service.cc:1467] Processing CreateTablet for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.687250 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.687489 2832 tablet_service.cc:1467] Processing CreateTablet for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.687834 2832 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.692687 2700 tablet_service.cc:1467] Processing CreateTablet for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:02.693132 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:02.696995 4139 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:02.698141 4140 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:02.700465 4140 tablet_bootstrap.cc:654] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.700779 4139 tablet_bootstrap.cc:654] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.702044 4138 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:02.703044 4141 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:02.703972 4139 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:02.704034 4138 tablet_bootstrap.cc:654] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.704608 4140 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:02.704757 4140 ts_tablet_manager.cc:1367] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
I20240502 14:16:02.705159 4139 ts_tablet_manager.cc:1367] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.003s
I20240502 14:16:02.705657 4140 raft_consensus.cc:348] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.705883 4140 raft_consensus.cc:374] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.706041 4140 raft_consensus.cc:729] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.706005 4139 raft_consensus.cc:348] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.706234 4139 raft_consensus.cc:374] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.706333 4139 raft_consensus.cc:729] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.706312 4140 consensus_queue.cc:260] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.706591 4139 consensus_queue.cc:260] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.706972 4140 ts_tablet_manager.cc:1398] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:16:02.707305 4140 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:02.707299 4139 ts_tablet_manager.cc:1398] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:02.707573 4139 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:02.708868 4140 tablet_bootstrap.cc:654] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.712294 4139 tablet_bootstrap.cc:654] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.713619 4138 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:02.713824 4138 ts_tablet_manager.cc:1367] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.012s user 0.010s sys 0.000s
I20240502 14:16:02.714756 4138 raft_consensus.cc:348] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.715106 4141 tablet_bootstrap.cc:654] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.715073 4138 raft_consensus.cc:374] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.715332 4138 raft_consensus.cc:729] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.715579 4138 consensus_queue.cc:260] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.716099 4138 ts_tablet_manager.cc:1398] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:16:02.716449 4138 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:02.718083 4138 tablet_bootstrap.cc:654] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.723577 4138 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:02.723776 4138 ts_tablet_manager.cc:1367] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20240502 14:16:02.723850 4140 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:02.724062 4140 ts_tablet_manager.cc:1367] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.017s user 0.008s sys 0.002s
I20240502 14:16:02.724598 4138 raft_consensus.cc:348] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.724882 4138 raft_consensus.cc:374] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.725003 4138 raft_consensus.cc:729] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.724953 4140 raft_consensus.cc:348] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.725261 4140 raft_consensus.cc:374] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.725391 4140 raft_consensus.cc:729] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.725306 4138 consensus_queue.cc:260] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.725719 4140 consensus_queue.cc:260] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.727116 4140 ts_tablet_manager.cc:1398] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:02.727366 4138 ts_tablet_manager.cc:1398] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:16:02.727494 4140 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:02.729976 4140 tablet_bootstrap.cc:654] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.731202 4141 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:02.731411 4141 ts_tablet_manager.cc:1367] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.029s user 0.008s sys 0.000s
I20240502 14:16:02.732209 4141 raft_consensus.cc:348] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.732430 4141 raft_consensus.cc:374] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.732532 4141 raft_consensus.cc:729] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.732760 4141 consensus_queue.cc:260] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.733585 4141 ts_tablet_manager.cc:1398] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:02.734536 4139 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:02.734733 4139 ts_tablet_manager.cc:1367] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.027s user 0.004s sys 0.003s
I20240502 14:16:02.735466 4139 raft_consensus.cc:348] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.735666 4139 raft_consensus.cc:374] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.735720 4139 raft_consensus.cc:729] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.735957 4139 consensus_queue.cc:260] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.736449 4139 ts_tablet_manager.cc:1398] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:16:02.736804 4139 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:02.737968 4140 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:02.738135 4140 ts_tablet_manager.cc:1367] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.002s
I20240502 14:16:02.739068 4140 raft_consensus.cc:348] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.739223 4139 tablet_bootstrap.cc:654] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:02.739300 4140 raft_consensus.cc:374] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.739442 4140 raft_consensus.cc:729] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.739683 4140 consensus_queue.cc:260] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.740243 4140 ts_tablet_manager.cc:1398] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:16:02.752676 4139 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:02.752836 4139 ts_tablet_manager.cc:1367] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.016s user 0.005s sys 0.008s
I20240502 14:16:02.753607 4139 raft_consensus.cc:348] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.753809 4139 raft_consensus.cc:374] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:02.753929 4139 raft_consensus.cc:729] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:02.754144 4139 consensus_queue.cc:260] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.755307 4139 ts_tablet_manager.cc:1398] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:02.846105 4065 raft_consensus.cc:482] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:02.846321 4065 raft_consensus.cc:504] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.847020 4065 leader_election.cc:288] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:02.847386 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7f7589fa40dd4af09cc87e6e793bcd8a" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:02.847631 2851 raft_consensus.cc:2454] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:02.847584 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7f7589fa40dd4af09cc87e6e793bcd8a" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:02.847808 2983 raft_consensus.cc:2454] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:02.848048 3049 leader_election.cc:302] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:02.848320 4065 raft_consensus.cc:2781] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:02.848425 4065 raft_consensus.cc:482] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:02.848512 4065 raft_consensus.cc:3037] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.850605 4065 raft_consensus.cc:504] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.851262 4065 leader_election.cc:288] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:02.851850 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7f7589fa40dd4af09cc87e6e793bcd8a" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:02.851853 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7f7589fa40dd4af09cc87e6e793bcd8a" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:02.852051 2982 raft_consensus.cc:3037] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.852088 2851 raft_consensus.cc:3037] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.854117 2851 raft_consensus.cc:2454] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:02.854116 2982 raft_consensus.cc:2454] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:02.854542 3049 leader_election.cc:302] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:02.854933 4065 raft_consensus.cc:2781] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:02.855096 4065 raft_consensus.cc:686] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Leader. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:16:02.855618 3877 raft_consensus.cc:482] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:02.855661 4065 consensus_queue.cc:237] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.855755 3877 raft_consensus.cc:504] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.856478 3877 leader_election.cc:288] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:02.856907 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "56aa6c21e5e947baaeaa6b42c89dfb8f" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:02.857141 3113 raft_consensus.cc:2454] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 0.
I20240502 14:16:02.857563 2788 leader_election.cc:302] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:02.857535 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "56aa6c21e5e947baaeaa6b42c89dfb8f" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:02.857761 2982 raft_consensus.cc:2454] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 0.
I20240502 14:16:02.857812 3877 raft_consensus.cc:2781] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:02.857918 3877 raft_consensus.cc:482] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:02.858003 3877 raft_consensus.cc:3037] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.859869 3877 raft_consensus.cc:504] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.859900 2581 catalog_manager.cc:5484] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:02.860419 3877 leader_election.cc:288] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:02.860805 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "56aa6c21e5e947baaeaa6b42c89dfb8f" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:02.861003 3113 raft_consensus.cc:3037] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.861042 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "56aa6c21e5e947baaeaa6b42c89dfb8f" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:02.861228 2983 raft_consensus.cc:3037] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.863055 3113 raft_consensus.cc:2454] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 1.
I20240502 14:16:02.863315 2983 raft_consensus.cc:2454] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6abc8535fe24d03bf71fb7092072fc4 in term 1.
I20240502 14:16:02.863415 2788 leader_election.cc:302] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:02.863755 3877 raft_consensus.cc:2781] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:02.864238 3877 raft_consensus.cc:686] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Becoming Leader. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Running, Role: LEADER
I20240502 14:16:02.864794 3877 consensus_queue.cc:237] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:02.867467 2581 catalog_manager.cc:5484] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: term changed from 0 to 1, leader changed from <none> to c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130). New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:02.875946 3747 raft_consensus.cc:482] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:02.876099 3747 raft_consensus.cc:504] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.876770 3747 leader_election.cc:288] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:02.877159 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9b5ec9436264dd9bd1f4e51bb1a5fb0" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:02.877399 2851 raft_consensus.cc:2454] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:02.877358 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9b5ec9436264dd9bd1f4e51bb1a5fb0" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:02.877669 2721 raft_consensus.cc:2454] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:02.877777 2919 leader_election.cc:302] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:02.878095 3747 raft_consensus.cc:2781] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:02.878222 3747 raft_consensus.cc:482] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:02.878309 3747 raft_consensus.cc:3037] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.880120 3747 raft_consensus.cc:504] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.880648 3747 leader_election.cc:288] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:02.880985 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9b5ec9436264dd9bd1f4e51bb1a5fb0" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:02.881095 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9b5ec9436264dd9bd1f4e51bb1a5fb0" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:02.881166 2851 raft_consensus.cc:3037] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.881304 2721 raft_consensus.cc:3037] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:02.883229 2721 raft_consensus.cc:2454] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:02.883576 2917 leader_election.cc:302] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:02.883862 3747 raft_consensus.cc:2781] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:02.884002 3747 raft_consensus.cc:686] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Leader. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:02.884284 3747 consensus_queue.cc:237] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:02.887027 2581 catalog_manager.cc:5484] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: term changed from 0 to 1, leader changed from <none> to 8cee803d912546e1981521fd8f16bb4e (127.27.220.131). New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:02.903496 2851 raft_consensus.cc:2454] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:03.264442 3877 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:03.275429 4155 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:03.275980 4155 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:16:03.283077 4152 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:03.289280 4155 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_5'
I20240502 14:16:03.354822 3747 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:03.365773 4071 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:03.397233 4065 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:03.416375 4065 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 329.445 ms
time per row: 0.164723 ms
I20240502 14:16:03.787192 2577 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:03.797565 2700 tablet_service.cc:1467] Processing CreateTablet for tablet 169109a7069249df905c760be1866c67 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.797909 2833 tablet_service.cc:1467] Processing CreateTablet for tablet 169109a7069249df905c760be1866c67 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.798094 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 169109a7069249df905c760be1866c67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.798051 3091 tablet_service.cc:1467] Processing CreateTablet for tablet 169109a7069249df905c760be1866c67 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.798430 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 169109a7069249df905c760be1866c67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.798475 3091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 169109a7069249df905c760be1866c67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.805652 2963 tablet_service.cc:1467] Processing CreateTablet for tablet e882402ebdb24b03a9c51d74dcfd3550 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.806210 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e882402ebdb24b03a9c51d74dcfd3550. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.809473 4222 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:03.812330 3092 tablet_service.cc:1467] Processing CreateTablet for tablet e882402ebdb24b03a9c51d74dcfd3550 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.812729 3092 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e882402ebdb24b03a9c51d74dcfd3550. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.817129 4225 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:03.817346 4224 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:03.817987 3093 tablet_service.cc:1467] Processing CreateTablet for tablet 28f1bc6021e04a05916d3e49a8c98597 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.818379 3093 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28f1bc6021e04a05916d3e49a8c98597. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.818847 4224 tablet_bootstrap.cc:654] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.818954 4225 tablet_bootstrap.cc:654] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.821058 2833 tablet_service.cc:1467] Processing CreateTablet for tablet 28f1bc6021e04a05916d3e49a8c98597 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.821507 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28f1bc6021e04a05916d3e49a8c98597. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.823110 4222 tablet_bootstrap.cc:654] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.824167 2702 tablet_service.cc:1467] Processing CreateTablet for tablet 28f1bc6021e04a05916d3e49a8c98597 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.824625 2702 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28f1bc6021e04a05916d3e49a8c98597. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.824945 4223 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:03.826830 4223 tablet_bootstrap.cc:654] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.829900 2700 tablet_service.cc:1467] Processing CreateTablet for tablet e882402ebdb24b03a9c51d74dcfd3550 (DEFAULT_TABLE table=rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:03.830318 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e882402ebdb24b03a9c51d74dcfd3550. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:03.831178 4223 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:03.831334 4223 ts_tablet_manager.cc:1367] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
I20240502 14:16:03.831511 4222 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:03.831657 4222 ts_tablet_manager.cc:1367] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.022s user 0.004s sys 0.000s
I20240502 14:16:03.832135 4223 raft_consensus.cc:348] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:03.832680 4223 raft_consensus.cc:374] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.832581 4222 raft_consensus.cc:348] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:03.832782 4223 raft_consensus.cc:729] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.832782 4222 raft_consensus.cc:374] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.833017 4223 consensus_queue.cc:260] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:03.832888 4222 raft_consensus.cc:729] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.834705 4223 ts_tablet_manager.cc:1398] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:16:03.835047 4223 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:03.834240 4222 consensus_queue.cc:260] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:03.837291 4225 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:03.837383 4222 ts_tablet_manager.cc:1398] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.006s user 0.003s sys 0.000s
I20240502 14:16:03.837529 4225 ts_tablet_manager.cc:1367] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.021s user 0.004s sys 0.000s
I20240502 14:16:03.837751 4222 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:03.838593 4225 raft_consensus.cc:348] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:03.838900 4225 raft_consensus.cc:374] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.839025 4225 raft_consensus.cc:729] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.839313 4225 consensus_queue.cc:260] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:03.840196 4225 ts_tablet_manager.cc:1398] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:03.842206 4223 tablet_bootstrap.cc:654] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.843154 4224 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:03.843346 4224 ts_tablet_manager.cc:1367] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.026s user 0.004s sys 0.000s
I20240502 14:16:03.844357 4224 raft_consensus.cc:348] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:03.844602 4224 raft_consensus.cc:374] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.844702 4224 raft_consensus.cc:729] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.845463 4222 tablet_bootstrap.cc:654] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.846158 4224 consensus_queue.cc:260] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:03.847095 4224 ts_tablet_manager.cc:1398] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:16:03.847409 4224 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:03.849385 4224 tablet_bootstrap.cc:654] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.854319 4222 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:03.854560 4222 ts_tablet_manager.cc:1367] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.017s user 0.001s sys 0.009s
I20240502 14:16:03.855031 4223 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:03.855197 4223 ts_tablet_manager.cc:1367] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.020s user 0.003s sys 0.000s
I20240502 14:16:03.855618 4222 raft_consensus.cc:348] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.855906 4222 raft_consensus.cc:374] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.856065 4222 raft_consensus.cc:729] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.855952 4223 raft_consensus.cc:348] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:03.856189 4223 raft_consensus.cc:374] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.856282 4223 raft_consensus.cc:729] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.856550 4222 consensus_queue.cc:260] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.856534 4223 consensus_queue.cc:260] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:03.857082 4223 ts_tablet_manager.cc:1398] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:03.857736 4223 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:03.858227 4224 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:03.858412 4224 ts_tablet_manager.cc:1367] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.006s
I20240502 14:16:03.859134 4224 raft_consensus.cc:348] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.859369 4224 raft_consensus.cc:374] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.859452 4224 raft_consensus.cc:729] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.859576 4223 tablet_bootstrap.cc:654] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.859683 4224 consensus_queue.cc:260] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.861007 4224 ts_tablet_manager.cc:1398] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:16:03.862301 4222 ts_tablet_manager.cc:1398] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.008s user 0.001s sys 0.001s
I20240502 14:16:03.862679 4223 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:03.862811 4223 ts_tablet_manager.cc:1367] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.003s
I20240502 14:16:03.863138 4222 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:03.865015 4222 tablet_bootstrap.cc:654] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:03.865468 4223 raft_consensus.cc:348] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.865757 4223 raft_consensus.cc:374] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.867619 4223 raft_consensus.cc:729] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.867925 4223 consensus_queue.cc:260] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.869302 4223 ts_tablet_manager.cc:1398] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 14:16:03.870756 4222 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:03.870951 4222 ts_tablet_manager.cc:1367] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
I20240502 14:16:03.871899 4222 raft_consensus.cc:348] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:03.872181 4222 raft_consensus.cc:374] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:03.872313 4222 raft_consensus.cc:729] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:03.872566 4222 consensus_queue.cc:260] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:03.873239 4222 ts_tablet_manager.cc:1398] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:03.888525 4186 raft_consensus.cc:482] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:03.888725 4186 raft_consensus.cc:504] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.889746 4186 leader_election.cc:288] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:03.890045 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28f1bc6021e04a05916d3e49a8c98597" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:03.890311 2851 raft_consensus.cc:2454] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:03.890305 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28f1bc6021e04a05916d3e49a8c98597" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:03.890578 2721 raft_consensus.cc:2454] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:03.890748 3049 leader_election.cc:302] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:03.891314 4186 raft_consensus.cc:2781] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:03.891455 4186 raft_consensus.cc:482] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:03.891558 4186 raft_consensus.cc:3037] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:03.895233 4186 raft_consensus.cc:504] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.896308 4186 leader_election.cc:288] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:03.896485 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28f1bc6021e04a05916d3e49a8c98597" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:03.896634 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "28f1bc6021e04a05916d3e49a8c98597" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:03.896726 2851 raft_consensus.cc:3037] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:03.896845 2721 raft_consensus.cc:3037] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:03.899566 2851 raft_consensus.cc:2454] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:03.899674 2721 raft_consensus.cc:2454] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:03.900056 3049 leader_election.cc:302] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:03.900476 4186 raft_consensus.cc:2781] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:03.900688 4186 raft_consensus.cc:686] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Leader. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:16:03.901173 4186 consensus_queue.cc:237] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:03.904567 2577 catalog_manager.cc:5484] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:04.085448 4119 raft_consensus.cc:482] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:04.085659 4119 raft_consensus.cc:504] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:04.086429 4119 leader_election.cc:288] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:04.086911 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:04.087023 3112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:04.087199 2982 raft_consensus.cc:2454] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:04.087318 3112 raft_consensus.cc:2454] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:04.087726 2655 leader_election.cc:302] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:04.088033 4119 raft_consensus.cc:2781] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:04.088153 4119 raft_consensus.cc:482] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:04.088258 4119 raft_consensus.cc:3037] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:04.091096 4119 raft_consensus.cc:504] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:04.091733 4119 leader_election.cc:288] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:04.092226 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:04.092468 2982 raft_consensus.cc:3037] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:04.092571 3112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:04.092794 3112 raft_consensus.cc:3037] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:04.094580 2982 raft_consensus.cc:2454] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:04.094746 3112 raft_consensus.cc:2454] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:04.094972 2655 leader_election.cc:302] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:04.095273 4119 raft_consensus.cc:2781] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:04.095635 4119 raft_consensus.cc:686] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:04.095892 4119 consensus_queue.cc:237] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:04.099009 2577 catalog_manager.cc:5484] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:04.184135 4119 raft_consensus.cc:482] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:04.184355 4119 raft_consensus.cc:504] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:04.184931 4119 leader_election.cc:288] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:04.185351 3112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "169109a7069249df905c760be1866c67" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:04.185427 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "169109a7069249df905c760be1866c67" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:04.185628 3112 raft_consensus.cc:2454] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:04.185668 2851 raft_consensus.cc:2454] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:04.186074 2657 leader_election.cc:302] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:04.186376 4119 raft_consensus.cc:2781] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:04.186522 4119 raft_consensus.cc:482] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:04.186652 4119 raft_consensus.cc:3037] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:04.189610 4119 raft_consensus.cc:504] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:04.190184 4119 leader_election.cc:288] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:04.190505 3112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "169109a7069249df905c760be1866c67" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:04.190673 3112 raft_consensus.cc:3037] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:04.190687 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "169109a7069249df905c760be1866c67" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:04.190888 2851 raft_consensus.cc:3037] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:04.192569 3112 raft_consensus.cc:2454] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:04.192862 2851 raft_consensus.cc:2454] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:04.192909 2657 leader_election.cc:302] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:04.193209 4119 raft_consensus.cc:2781] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:04.193388 4119 raft_consensus.cc:686] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:04.193684 4119 consensus_queue.cc:237] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:04.196364 2577 catalog_manager.cc:5484] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:04.362620 4186 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:04.368633 4186 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:04.488662 4119 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:04.494400 4119 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:04.626528 4239 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:04.626823 4239 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:04.633930 4239 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:16:04.648795 4246 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:04.653133 4236 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_6'
INSERT report
rows total: 2000
time total: 253.454 ms
time per row: 0.126727 ms
I20240502 14:16:05.038386 2581 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_7"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:05.046455 2963 tablet_service.cc:1467] Processing CreateTablet for tablet 6f785b6443b741a483e16eb154439b27 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.046685 2833 tablet_service.cc:1467] Processing CreateTablet for tablet 6f785b6443b741a483e16eb154439b27 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.047014 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6f785b6443b741a483e16eb154439b27. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.047118 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6f785b6443b741a483e16eb154439b27. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.048667 2961 tablet_service.cc:1467] Processing CreateTablet for tablet 24027cd85c0d4480bd1cd4639f4f6540 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.049052 2961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 24027cd85c0d4480bd1cd4639f4f6540. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.048947 3091 tablet_service.cc:1467] Processing CreateTablet for tablet 24027cd85c0d4480bd1cd4639f4f6540 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.049365 3091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 24027cd85c0d4480bd1cd4639f4f6540. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.049891 3093 tablet_service.cc:1467] Processing CreateTablet for tablet 609b87df71e64919a29a4fa093dfca23 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.050246 3093 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 609b87df71e64919a29a4fa093dfca23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.053920 2962 tablet_service.cc:1467] Processing CreateTablet for tablet 609b87df71e64919a29a4fa093dfca23 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.054275 2962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 609b87df71e64919a29a4fa093dfca23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.056345 4301 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:05.057694 2700 tablet_service.cc:1467] Processing CreateTablet for tablet 6f785b6443b741a483e16eb154439b27 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.058161 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6f785b6443b741a483e16eb154439b27. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.061949 2831 tablet_service.cc:1467] Processing CreateTablet for tablet 24027cd85c0d4480bd1cd4639f4f6540 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.062333 2831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 24027cd85c0d4480bd1cd4639f4f6540. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.065711 4302 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:05.066090 4300 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:05.066416 2702 tablet_service.cc:1467] Processing CreateTablet for tablet 609b87df71e64919a29a4fa093dfca23 (DEFAULT_TABLE table=rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:05.066848 2702 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 609b87df71e64919a29a4fa093dfca23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:05.067672 4300 tablet_bootstrap.cc:654] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.068643 4302 tablet_bootstrap.cc:654] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.079391 4302 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:05.079624 4302 ts_tablet_manager.cc:1367] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.014s user 0.003s sys 0.007s
I20240502 14:16:05.080461 4302 raft_consensus.cc:348] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.080847 4302 raft_consensus.cc:374] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.081010 4302 raft_consensus.cc:729] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.081373 4302 consensus_queue.cc:260] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.082216 4301 tablet_bootstrap.cc:654] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.083292 4305 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:05.085561 4305 tablet_bootstrap.cc:654] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.089542 4302 ts_tablet_manager.cc:1398] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.010s user 0.002s sys 0.000s
I20240502 14:16:05.094331 4302 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:05.094761 4300 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:05.094969 4300 ts_tablet_manager.cc:1367] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.029s user 0.011s sys 0.016s
I20240502 14:16:05.096081 4300 raft_consensus.cc:348] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.096400 4300 raft_consensus.cc:374] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.096524 4300 raft_consensus.cc:729] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.096791 4300 consensus_queue.cc:260] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.097266 4302 tablet_bootstrap.cc:654] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.097602 4300 ts_tablet_manager.cc:1398] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 14:16:05.097923 4300 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:05.100450 4300 tablet_bootstrap.cc:654] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.106601 4302 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:05.106599 4305 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:05.106742 4305 ts_tablet_manager.cc:1367] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.024s user 0.006s sys 0.006s
I20240502 14:16:05.106772 4302 ts_tablet_manager.cc:1367] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.013s user 0.005s sys 0.001s
I20240502 14:16:05.107451 4302 raft_consensus.cc:348] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.107627 4302 raft_consensus.cc:374] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.107533 4305 raft_consensus.cc:348] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.107707 4302 raft_consensus.cc:729] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.107715 4305 raft_consensus.cc:374] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.107795 4305 raft_consensus.cc:729] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.107955 4302 consensus_queue.cc:260] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.108002 4305 consensus_queue.cc:260] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.108472 4305 ts_tablet_manager.cc:1398] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:05.108497 4302 ts_tablet_manager.cc:1398] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:05.108743 4305 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:05.110198 4301 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:05.110260 4305 tablet_bootstrap.cc:654] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.110359 4301 ts_tablet_manager.cc:1367] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.054s user 0.009s sys 0.005s
I20240502 14:16:05.111105 4301 raft_consensus.cc:348] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.111299 4301 raft_consensus.cc:374] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.111402 4301 raft_consensus.cc:729] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.111670 4301 consensus_queue.cc:260] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.112313 4301 ts_tablet_manager.cc:1398] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:16:05.112751 4301 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:05.114351 4301 tablet_bootstrap.cc:654] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.116382 4300 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:05.116603 4300 ts_tablet_manager.cc:1367] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.019s user 0.006s sys 0.001s
I20240502 14:16:05.117398 4300 raft_consensus.cc:348] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.117707 4300 raft_consensus.cc:374] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.117818 4300 raft_consensus.cc:729] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.118109 4300 consensus_queue.cc:260] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.118775 4300 ts_tablet_manager.cc:1398] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:05.119119 4300 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:05.120568 4300 tablet_bootstrap.cc:654] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:05.121352 4305 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:05.121568 4305 ts_tablet_manager.cc:1367] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.000s
I20240502 14:16:05.122328 4305 raft_consensus.cc:348] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.123256 4305 raft_consensus.cc:374] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.123394 4305 raft_consensus.cc:729] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.123611 4305 consensus_queue.cc:260] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.124083 4305 ts_tablet_manager.cc:1398] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 14:16:05.125113 4300 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:05.125341 4300 ts_tablet_manager.cc:1367] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
I20240502 14:16:05.125973 4300 raft_consensus.cc:348] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.126140 4300 raft_consensus.cc:374] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.126241 4300 raft_consensus.cc:729] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.126453 4300 consensus_queue.cc:260] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.126910 4300 ts_tablet_manager.cc:1398] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 14:16:05.129858 4301 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:05.130048 4301 ts_tablet_manager.cc:1367] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.017s user 0.008s sys 0.004s
I20240502 14:16:05.130609 4301 raft_consensus.cc:348] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.130842 4301 raft_consensus.cc:374] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:05.130925 4301 raft_consensus.cc:729] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:05.131104 4301 consensus_queue.cc:260] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.131549 4301 ts_tablet_manager.cc:1398] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 14:16:05.143325 4216 raft_consensus.cc:482] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:05.143468 4216 raft_consensus.cc:504] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.144101 4216 leader_election.cc:288] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:05.144567 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "24027cd85c0d4480bd1cd4639f4f6540" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:05.144623 3112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "24027cd85c0d4480bd1cd4639f4f6540" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:05.144769 2851 raft_consensus.cc:2454] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:05.144868 3112 raft_consensus.cc:2454] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:05.145239 2919 leader_election.cc:302] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:05.145588 4216 raft_consensus.cc:2781] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:05.145722 4216 raft_consensus.cc:482] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:05.145824 4216 raft_consensus.cc:3037] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.147769 4216 raft_consensus.cc:504] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.148273 4216 leader_election.cc:288] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:05.148690 3112 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "24027cd85c0d4480bd1cd4639f4f6540" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:05.148716 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "24027cd85c0d4480bd1cd4639f4f6540" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:05.148883 3112 raft_consensus.cc:3037] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.148882 2851 raft_consensus.cc:3037] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.150770 2851 raft_consensus.cc:2454] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:05.150794 3112 raft_consensus.cc:2454] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:05.151130 2919 leader_election.cc:302] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:05.151379 4216 raft_consensus.cc:2781] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:05.151741 4216 raft_consensus.cc:686] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Leader. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:05.152019 4216 consensus_queue.cc:237] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:05.154955 2579 catalog_manager.cc:5484] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: term changed from 0 to 1, leader changed from <none> to 8cee803d912546e1981521fd8f16bb4e (127.27.220.131). New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:05.244546 4119 raft_consensus.cc:482] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:05.244728 4119 raft_consensus.cc:504] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.245358 4119 leader_election.cc:288] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:05.245836 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6f785b6443b741a483e16eb154439b27" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:05.245877 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6f785b6443b741a483e16eb154439b27" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:05.246066 2851 raft_consensus.cc:2454] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:05.246150 2982 raft_consensus.cc:2454] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:05.246448 2658 leader_election.cc:302] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:05.246752 4119 raft_consensus.cc:2781] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:05.246878 4119 raft_consensus.cc:482] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:05.246971 4119 raft_consensus.cc:3037] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.249033 4119 raft_consensus.cc:504] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.250018 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6f785b6443b741a483e16eb154439b27" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:05.249994 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6f785b6443b741a483e16eb154439b27" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:05.250205 2982 raft_consensus.cc:3037] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.250212 2851 raft_consensus.cc:3037] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.250320 4119 leader_election.cc:288] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:05.253033 2851 raft_consensus.cc:2454] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:05.253176 2982 raft_consensus.cc:2454] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:05.253456 2658 leader_election.cc:302] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:05.253859 4119 raft_consensus.cc:2781] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:05.254027 4119 raft_consensus.cc:686] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:05.254272 4119 consensus_queue.cc:237] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.255210 4186 raft_consensus.cc:482] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:05.255393 4186 raft_consensus.cc:504] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.255997 4186 leader_election.cc:288] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:05.256639 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "609b87df71e64919a29a4fa093dfca23" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:05.256883 2721 raft_consensus.cc:2454] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:05.257364 3047 leader_election.cc:302] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:05.257702 4186 raft_consensus.cc:2781] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:05.257790 4186 raft_consensus.cc:482] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:05.257892 4186 raft_consensus.cc:3037] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.257696 2579 catalog_manager.cc:5484] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:05.259969 4186 raft_consensus.cc:504] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.260923 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "609b87df71e64919a29a4fa093dfca23" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:05.260957 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "609b87df71e64919a29a4fa093dfca23" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:05.261181 2982 raft_consensus.cc:2454] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 0.
I20240502 14:16:05.261811 4186 leader_election.cc:288] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:05.262250 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "609b87df71e64919a29a4fa093dfca23" candidate_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:05.262415 2721 raft_consensus.cc:3037] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.264616 2721 raft_consensus.cc:2454] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 472a7cc2e2d04cf0b36f7e43ed55f1d2 in term 1.
I20240502 14:16:05.264958 3047 leader_election.cc:302] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters: 8cee803d912546e1981521fd8f16bb4e
I20240502 14:16:05.265202 4186 raft_consensus.cc:2781] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:05.265364 4186 raft_consensus.cc:686] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Leader. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:16:05.265622 4186 consensus_queue.cc:237] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:05.268872 2579 catalog_manager.cc:5484] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: term changed from 0 to 1, leader changed from <none> to 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132). New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:05.652290 4216 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:05.660152 4216 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:05.660480 4317 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:05.660975 4317 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:16:05.665374 2982 raft_consensus.cc:3037] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:05.670022 4295 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:05.670301 4317 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:16:05.678783 4186 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_7'
I20240502 14:16:05.727613 4283 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:05.737258 4119 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 285.334 ms
time per row: 0.142667 ms
I20240502 14:16:06.127032 2577 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_8"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:06.135869 2702 tablet_service.cc:1467] Processing CreateTablet for tablet 145a6a19b8474746a30ad51563419f1c (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.136380 2702 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 145a6a19b8474746a30ad51563419f1c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.136400 2831 tablet_service.cc:1467] Processing CreateTablet for tablet 145a6a19b8474746a30ad51563419f1c (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.136582 2963 tablet_service.cc:1467] Processing CreateTablet for tablet 145a6a19b8474746a30ad51563419f1c (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.136794 2831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 145a6a19b8474746a30ad51563419f1c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.137148 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 145a6a19b8474746a30ad51563419f1c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.141678 4389 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:06.142282 3093 tablet_service.cc:1467] Processing CreateTablet for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.142711 3093 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.143508 2700 tablet_service.cc:1467] Processing CreateTablet for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.143752 2701 tablet_service.cc:1467] Processing CreateTablet for tablet a1a32a28425e442ca4846e423e457345 (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.143893 2700 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.144289 4389 tablet_bootstrap.cc:654] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.145774 2701 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1a32a28425e442ca4846e423e457345. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.145983 3091 tablet_service.cc:1467] Processing CreateTablet for tablet a1a32a28425e442ca4846e423e457345 (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.146441 3091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1a32a28425e442ca4846e423e457345. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.148236 4390 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:06.150575 4390 tablet_bootstrap.cc:654] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.151016 2962 tablet_service.cc:1467] Processing CreateTablet for tablet a1a32a28425e442ca4846e423e457345 (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.151535 2962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1a32a28425e442ca4846e423e457345. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.152537 2833 tablet_service.cc:1467] Processing CreateTablet for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 (DEFAULT_TABLE table=rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:06.153050 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:06.159067 4390 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:06.159231 4390 ts_tablet_manager.cc:1367] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.006s
I20240502 14:16:06.160084 4390 raft_consensus.cc:348] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.160318 4390 raft_consensus.cc:374] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.160404 4390 raft_consensus.cc:729] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.160651 4390 consensus_queue.cc:260] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.161274 4390 ts_tablet_manager.cc:1398] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:06.166540 4390 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:06.168674 4390 tablet_bootstrap.cc:654] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.174108 4389 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:06.174350 4389 ts_tablet_manager.cc:1367] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.033s user 0.021s sys 0.007s
I20240502 14:16:06.175314 4389 raft_consensus.cc:348] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.175640 4389 raft_consensus.cc:374] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.175801 4389 raft_consensus.cc:729] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.176127 4389 consensus_queue.cc:260] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.178807 4389 ts_tablet_manager.cc:1398] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 14:16:06.179107 4389 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:06.179594 4325 raft_consensus.cc:482] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:06.179735 4325 raft_consensus.cc:504] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.180457 4325 leader_election.cc:288] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:06.181243 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:06.181486 2721 raft_consensus.cc:2454] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:06.182487 2917 leader_election.cc:302] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:06.183964 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:06.184782 4325 raft_consensus.cc:2781] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:06.184913 4325 raft_consensus.cc:482] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:06.185021 4325 raft_consensus.cc:3037] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
W20240502 14:16:06.184952 2919 leader_election.cc:341] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:16:06.188017 4325 raft_consensus.cc:504] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.189286 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:06.189496 2721 raft_consensus.cc:3037] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:06.191118 4391 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:06.192090 4325 leader_election.cc:288] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Requested vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:06.192580 2721 raft_consensus.cc:2454] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:06.192617 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
W20240502 14:16:06.193249 2919 leader_election.cc:341] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:16:06.193446 2917 leader_election.cc:302] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e; no voters: c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:16:06.194039 4325 raft_consensus.cc:2781] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:06.194073 4395 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:06.194584 4325 raft_consensus.cc:686] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Leader. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:06.194947 4325 consensus_queue.cc:237] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.194605 4389 tablet_bootstrap.cc:654] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.196344 4395 tablet_bootstrap.cc:654] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.197077 4391 tablet_bootstrap.cc:654] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.199409 4390 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:06.199636 4390 ts_tablet_manager.cc:1367] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.033s user 0.008s sys 0.024s
I20240502 14:16:06.201676 4391 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:06.201933 4391 ts_tablet_manager.cc:1367] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.002s
I20240502 14:16:06.201710 2573 catalog_manager.cc:5484] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e reported cstate change: term changed from 0 to 1, leader changed from <none> to 8cee803d912546e1981521fd8f16bb4e (127.27.220.131). New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:06.204010 4390 raft_consensus.cc:348] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.204267 4390 raft_consensus.cc:374] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.204455 4389 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:06.204380 4390 raft_consensus.cc:729] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.204669 4389 ts_tablet_manager.cc:1367] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.026s user 0.000s sys 0.006s
I20240502 14:16:06.204766 4390 consensus_queue.cc:260] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.205654 4389 raft_consensus.cc:348] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.204913 4391 raft_consensus.cc:348] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.206241 4391 raft_consensus.cc:374] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.206437 4391 raft_consensus.cc:729] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.206852 4391 consensus_queue.cc:260] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.207705 4391 ts_tablet_manager.cc:1398] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.006s user 0.001s sys 0.001s
I20240502 14:16:06.207886 4390 ts_tablet_manager.cc:1398] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.008s user 0.000s sys 0.001s
I20240502 14:16:06.208060 4391 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:06.208173 4389 raft_consensus.cc:374] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.208276 4390 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:06.208277 4389 raft_consensus.cc:729] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.208539 4389 consensus_queue.cc:260] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.210558 4390 tablet_bootstrap.cc:654] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.210960 4391 tablet_bootstrap.cc:654] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.215018 4389 ts_tablet_manager.cc:1398] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.010s user 0.002s sys 0.000s
I20240502 14:16:06.219056 4395 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:06.219161 4391 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:06.219235 4395 ts_tablet_manager.cc:1367] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.025s user 0.003s sys 0.013s
I20240502 14:16:06.219357 4391 ts_tablet_manager.cc:1367] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.011s user 0.009s sys 0.000s
I20240502 14:16:06.219810 4390 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:06.219972 4390 ts_tablet_manager.cc:1367] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.012s user 0.007s sys 0.002s
I20240502 14:16:06.219897 4395 raft_consensus.cc:348] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.220082 4395 raft_consensus.cc:374] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.220178 4395 raft_consensus.cc:729] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.220161 4391 raft_consensus.cc:348] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.220423 4391 raft_consensus.cc:374] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.220422 4395 consensus_queue.cc:260] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.220528 4391 raft_consensus.cc:729] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.220785 4390 raft_consensus.cc:348] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.220774 4391 consensus_queue.cc:260] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.221024 4390 raft_consensus.cc:374] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.221143 4390 raft_consensus.cc:729] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.221474 4391 ts_tablet_manager.cc:1398] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:06.221407 4390 consensus_queue.cc:260] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.221999 4390 ts_tablet_manager.cc:1398] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:06.222308 4395 ts_tablet_manager.cc:1398] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:06.222600 4395 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:06.223987 4395 tablet_bootstrap.cc:654] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:06.226505 4395 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:06.226699 4395 ts_tablet_manager.cc:1367] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:16:06.227428 4395 raft_consensus.cc:348] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.227623 4395 raft_consensus.cc:374] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:06.227703 4395 raft_consensus.cc:729] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:06.227960 4395 consensus_queue.cc:260] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.228515 4395 ts_tablet_manager.cc:1398] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:06.270749 4283 raft_consensus.cc:482] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:06.270936 4283 raft_consensus.cc:504] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.271569 4283 leader_election.cc:288] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:06.271991 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:06.272169 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:06.272289 3113 raft_consensus.cc:2454] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:06.272433 2852 raft_consensus.cc:2454] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:06.272830 2657 leader_election.cc:302] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:06.273115 4283 raft_consensus.cc:2781] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:06.273229 4283 raft_consensus.cc:482] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:06.273344 4283 raft_consensus.cc:3037] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:06.275656 4283 raft_consensus.cc:504] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.276365 4283 leader_election.cc:288] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:06.276746 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:06.276877 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:06.277031 3113 raft_consensus.cc:3037] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:06.277097 2852 raft_consensus.cc:3037] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:06.280138 3113 raft_consensus.cc:2454] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:06.280179 2852 raft_consensus.cc:2454] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:06.280665 2657 leader_election.cc:302] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:06.280967 4283 raft_consensus.cc:2781] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:06.281122 4283 raft_consensus.cc:686] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:06.281392 4283 consensus_queue.cc:237] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } }
I20240502 14:16:06.284368 2573 catalog_manager.cc:5484] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:06.361923 4325 raft_consensus.cc:482] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:06.362129 4325 raft_consensus.cc:504] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.362761 4325 leader_election.cc:288] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:06.363345 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1a32a28425e442ca4846e423e457345" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:06.363377 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1a32a28425e442ca4846e423e457345" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:06.363595 3113 raft_consensus.cc:2454] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:06.363616 2721 raft_consensus.cc:2454] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:06.364037 2918 leader_election.cc:302] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:06.364328 4325 raft_consensus.cc:2781] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:06.364410 4325 raft_consensus.cc:482] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:06.364529 4325 raft_consensus.cc:3037] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:06.366418 4325 raft_consensus.cc:504] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.366951 4325 leader_election.cc:288] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:06.367339 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1a32a28425e442ca4846e423e457345" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:06.367420 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a1a32a28425e442ca4846e423e457345" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:06.367532 3113 raft_consensus.cc:3037] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:06.367638 2721 raft_consensus.cc:3037] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:06.369267 3113 raft_consensus.cc:2454] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:06.369377 2721 raft_consensus.cc:2454] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:06.369660 2918 leader_election.cc:302] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:06.369941 4325 raft_consensus.cc:2781] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:06.370088 4325 raft_consensus.cc:686] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Leader. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:06.370357 4325 consensus_queue.cc:237] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:06.372993 2573 catalog_manager.cc:5484] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: term changed from 0 to 1, leader changed from <none> to 8cee803d912546e1981521fd8f16bb4e (127.27.220.131). New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:06.567706 4318 raft_consensus.cc:482] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:06.567903 4318 raft_consensus.cc:504] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } }
I20240502 14:16:06.568550 4318 leader_election.cc:288] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:06.569124 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:06.569167 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "c6abc8535fe24d03bf71fb7092072fc4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6b3499bcb84b45a094f7166639704ed4" is_pre_election: true
I20240502 14:16:06.569398 2721 raft_consensus.cc:2379] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c6abc8535fe24d03bf71fb7092072fc4 in current term 1: Already voted for candidate 8cee803d912546e1981521fd8f16bb4e in this term.
I20240502 14:16:06.569902 2787 leader_election.cc:302] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: c6abc8535fe24d03bf71fb7092072fc4; no voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e
I20240502 14:16:06.570246 4318 raft_consensus.cc:3037] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:06.572098 4318 raft_consensus.cc:2726] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:16:06.716565 4325 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:06.741139 4325 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:06.785606 4405 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:06.785997 4405 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:06.793951 4405 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:16:06.795578 4377 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:06.800215 4283 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_8'
I20240502 14:16:06.958796 2721 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:16:06.959327 3113 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:16:06.959714 4323 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:06.960135 4409 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:06.997110 4446 mvcc.cc:205] Tried to move back new op lower bound from 7023244767053705216 to 7023244764654837760. Current Snapshot: MvccSnapshot[applied={T|T < 7023244767053705216}]
I20240502 14:16:07.014940 4442 mvcc.cc:205] Tried to move back new op lower bound from 7023244767053705216 to 7023244764654837760. Current Snapshot: MvccSnapshot[applied={T|T < 7023244767053705216}]
INSERT report
rows total: 2000
time total: 279.817 ms
time per row: 0.139909 ms
I20240502 14:16:07.222105 2577 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:46436:
name: "rebalance_test_table_9"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:07.231483 2831 tablet_service.cc:1467] Processing CreateTablet for tablet 3cc7001d1b9b425a8758f8e15914c289 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.231595 2962 tablet_service.cc:1467] Processing CreateTablet for tablet 3cc7001d1b9b425a8758f8e15914c289 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.231791 2833 tablet_service.cc:1467] Processing CreateTablet for tablet 33e74e4a8ff147d69a8051f0a898fae6 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.231954 2831 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3cc7001d1b9b425a8758f8e15914c289. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.232097 2962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3cc7001d1b9b425a8758f8e15914c289. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.232152 2833 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 33e74e4a8ff147d69a8051f0a898fae6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.234211 3091 tablet_service.cc:1467] Processing CreateTablet for tablet 33e74e4a8ff147d69a8051f0a898fae6 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.234612 3091 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 33e74e4a8ff147d69a8051f0a898fae6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.236450 2961 tablet_service.cc:1467] Processing CreateTablet for tablet 81be921c89194d3bb9c3d41be938ed74 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.236855 2961 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81be921c89194d3bb9c3d41be938ed74. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.237936 3093 tablet_service.cc:1467] Processing CreateTablet for tablet 3cc7001d1b9b425a8758f8e15914c289 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.238368 3093 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3cc7001d1b9b425a8758f8e15914c289. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.238356 2701 tablet_service.cc:1467] Processing CreateTablet for tablet 81be921c89194d3bb9c3d41be938ed74 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.238792 2701 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81be921c89194d3bb9c3d41be938ed74. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.241991 2963 tablet_service.cc:1467] Processing CreateTablet for tablet 33e74e4a8ff147d69a8051f0a898fae6 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.242477 2963 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 33e74e4a8ff147d69a8051f0a898fae6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.245757 2832 tablet_service.cc:1467] Processing CreateTablet for tablet 81be921c89194d3bb9c3d41be938ed74 (DEFAULT_TABLE table=rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:07.246145 2832 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81be921c89194d3bb9c3d41be938ed74. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:07.246716 4465 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:07.246762 4466 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: Bootstrap starting.
I20240502 14:16:07.248891 4465 tablet_bootstrap.cc:654] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.249459 4466 tablet_bootstrap.cc:654] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.257853 4465 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:07.258020 4465 ts_tablet_manager.cc:1367] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.011s user 0.009s sys 0.000s
I20240502 14:16:07.258888 4465 raft_consensus.cc:348] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.259094 4465 raft_consensus.cc:374] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.259199 4465 raft_consensus.cc:729] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.259310 4466 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: No bootstrap required, opened a new log
I20240502 14:16:07.259469 4466 ts_tablet_manager.cc:1367] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.006s
I20240502 14:16:07.259454 4465 consensus_queue.cc:260] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.260282 4466 raft_consensus.cc:348] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.260833 4465 ts_tablet_manager.cc:1398] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:16:07.261163 4465 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:07.262560 4466 raft_consensus.cc:374] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.262708 4466 raft_consensus.cc:729] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.263803 4465 tablet_bootstrap.cc:654] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.263029 4466 consensus_queue.cc:260] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.267467 4466 ts_tablet_manager.cc:1398] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: Time spent starting tablet: real 0.008s user 0.001s sys 0.002s
I20240502 14:16:07.268230 4467 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:07.271163 4467 tablet_bootstrap.cc:654] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.271924 4465 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:07.272192 4465 ts_tablet_manager.cc:1367] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.007s
I20240502 14:16:07.273151 4465 raft_consensus.cc:348] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.273504 4465 raft_consensus.cc:374] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.273613 4470 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:07.273660 4465 raft_consensus.cc:729] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.274032 4465 consensus_queue.cc:260] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.275388 4470 tablet_bootstrap.cc:654] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.275559 4465 ts_tablet_manager.cc:1398] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:07.275895 4465 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e: Bootstrap starting.
I20240502 14:16:07.279047 4465 tablet_bootstrap.cc:654] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.282895 4470 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:07.283044 4470 ts_tablet_manager.cc:1367] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.003s
I20240502 14:16:07.283187 4467 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:07.283363 4467 ts_tablet_manager.cc:1367] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.006s
I20240502 14:16:07.283720 4470 raft_consensus.cc:348] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.283900 4470 raft_consensus.cc:374] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.283998 4470 raft_consensus.cc:729] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.284039 4467 raft_consensus.cc:348] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.284225 4467 raft_consensus.cc:374] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.284294 4467 raft_consensus.cc:729] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.284222 4470 consensus_queue.cc:260] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.284492 4467 consensus_queue.cc:260] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.284957 4467 ts_tablet_manager.cc:1398] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 14:16:07.285449 4470 ts_tablet_manager.cc:1398] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:16:07.285751 4470 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Bootstrap starting.
I20240502 14:16:07.285971 4465 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e: No bootstrap required, opened a new log
I20240502 14:16:07.286140 4465 ts_tablet_manager.cc:1367] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.002s
I20240502 14:16:07.286988 4465 raft_consensus.cc:348] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.287230 4467 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:07.287241 4465 raft_consensus.cc:374] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.287349 4465 raft_consensus.cc:729] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.287583 4470 tablet_bootstrap.cc:654] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.287597 4465 consensus_queue.cc:260] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.288991 4465 ts_tablet_manager.cc:1398] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 14:16:07.290685 4467 tablet_bootstrap.cc:654] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.295630 4470 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: No bootstrap required, opened a new log
I20240502 14:16:07.295898 4470 ts_tablet_manager.cc:1367] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.006s
I20240502 14:16:07.297168 4470 raft_consensus.cc:348] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.297506 4470 raft_consensus.cc:374] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.297668 4470 raft_consensus.cc:729] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.297865 4467 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:07.298059 4467 ts_tablet_manager.cc:1367] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.011s user 0.001s sys 0.005s
I20240502 14:16:07.297987 4470 consensus_queue.cc:260] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.298676 4470 ts_tablet_manager.cc:1398] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:07.298833 4467 raft_consensus.cc:348] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.299052 4467 raft_consensus.cc:374] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.299154 4467 raft_consensus.cc:729] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.299454 4467 consensus_queue.cc:260] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.300369 4467 ts_tablet_manager.cc:1398] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:16:07.300676 4467 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: Bootstrap starting.
I20240502 14:16:07.302181 4467 tablet_bootstrap.cc:654] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:07.310539 4467 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: No bootstrap required, opened a new log
I20240502 14:16:07.310707 4467 ts_tablet_manager.cc:1367] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.002s
I20240502 14:16:07.311869 4467 raft_consensus.cc:348] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.312089 4467 raft_consensus.cc:374] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:07.312201 4467 raft_consensus.cc:729] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6abc8535fe24d03bf71fb7092072fc4, State: Initialized, Role: FOLLOWER
I20240502 14:16:07.312424 4467 consensus_queue.cc:260] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.314549 4467 ts_tablet_manager.cc:1398] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240502 14:16:07.430454 4460 raft_consensus.cc:482] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:07.430656 4460 raft_consensus.cc:504] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.431363 4460 leader_election.cc:288] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:07.431963 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "33e74e4a8ff147d69a8051f0a898fae6" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:07.431963 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "33e74e4a8ff147d69a8051f0a898fae6" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:07.432225 3113 raft_consensus.cc:2454] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:07.432224 2851 raft_consensus.cc:2454] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:07.432731 2918 leader_election.cc:302] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:07.433002 4460 raft_consensus.cc:2781] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:07.433120 4460 raft_consensus.cc:482] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:07.433192 4460 raft_consensus.cc:3037] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.434589 4377 raft_consensus.cc:482] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:07.434775 4377 raft_consensus.cc:504] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.435313 4460 raft_consensus.cc:504] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.435427 4377 leader_election.cc:288] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:07.435942 4460 leader_election.cc:288] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:07.435951 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81be921c89194d3bb9c3d41be938ed74" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:07.436201 2851 raft_consensus.cc:2454] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:07.436353 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81be921c89194d3bb9c3d41be938ed74" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e" is_pre_election: true
I20240502 14:16:07.436445 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "33e74e4a8ff147d69a8051f0a898fae6" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:07.436476 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "33e74e4a8ff147d69a8051f0a898fae6" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:07.436576 2982 raft_consensus.cc:2454] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 0.
I20240502 14:16:07.436652 2852 raft_consensus.cc:3037] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.436666 3113 raft_consensus.cc:3037] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.437067 2658 leader_election.cc:302] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:07.437427 4377 raft_consensus.cc:2781] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:07.437561 4377 raft_consensus.cc:482] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:07.437664 4377 raft_consensus.cc:3037] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.438930 2852 raft_consensus.cc:2454] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:07.439303 2919 leader_election.cc:302] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 8cee803d912546e1981521fd8f16bb4e, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:07.439392 3113 raft_consensus.cc:2454] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:07.439596 4460 raft_consensus.cc:2781] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:07.439801 4460 raft_consensus.cc:686] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Leader. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:07.440099 4460 consensus_queue.cc:237] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.440236 4377 raft_consensus.cc:504] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.441053 4377 leader_election.cc:288] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:07.441169 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81be921c89194d3bb9c3d41be938ed74" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:07.441361 2852 raft_consensus.cc:3037] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.441982 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "81be921c89194d3bb9c3d41be938ed74" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:07.442157 2982 raft_consensus.cc:3037] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.443476 4460 raft_consensus.cc:482] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:16:07.443715 2852 raft_consensus.cc:2454] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:07.443847 4460 raft_consensus.cc:504] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.444219 2658 leader_election.cc:302] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:07.444497 2982 raft_consensus.cc:2454] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 1.
I20240502 14:16:07.444579 4460 leader_election.cc:288] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:07.444851 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" is_pre_election: true
I20240502 14:16:07.445080 3113 raft_consensus.cc:2454] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:07.445302 4377 raft_consensus.cc:2781] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:07.445513 2918 leader_election.cc:302] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:07.445475 4377 raft_consensus.cc:686] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:07.445143 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4" is_pre_election: true
I20240502 14:16:07.446038 2852 raft_consensus.cc:2454] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 0.
I20240502 14:16:07.446023 4377 consensus_queue.cc:237] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } }
I20240502 14:16:07.446120 2577 catalog_manager.cc:5484] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: term changed from 0 to 1, leader changed from <none> to 8cee803d912546e1981521fd8f16bb4e (127.27.220.131). New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:07.446537 4460 raft_consensus.cc:2781] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:16:07.446667 4460 raft_consensus.cc:482] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:16:07.446806 4460 raft_consensus.cc:3037] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.448897 4460 raft_consensus.cc:504] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.449889 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:07.450026 3113 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289" candidate_uuid: "8cee803d912546e1981521fd8f16bb4e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:07.450156 2852 raft_consensus.cc:3037] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.450237 3113 raft_consensus.cc:3037] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:07.450330 2579 catalog_manager.cc:5484] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:07.452363 3113 raft_consensus.cc:2454] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:07.452402 2852 raft_consensus.cc:2454] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8cee803d912546e1981521fd8f16bb4e in term 1.
I20240502 14:16:07.452710 4460 leader_election.cc:288] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:07.452781 2918 leader_election.cc:302] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:07.452994 4460 raft_consensus.cc:2781] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:07.453260 4460 raft_consensus.cc:686] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Leader. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:07.453497 4460 consensus_queue.cc:237] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } }
I20240502 14:16:07.456233 2579 catalog_manager.cc:5484] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: term changed from 0 to 1, leader changed from <none> to 8cee803d912546e1981521fd8f16bb4e (127.27.220.131). New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:07.822135 4460 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:07.828307 4460 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:16:07.828536 4481 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:07.828948 4481 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:07.840965 4481 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_9'
I20240502 14:16:07.922215 4325 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:07.928136 4283 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:07.938854 4460 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:16:08.000517 4377 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 264.035 ms
time per row: 0.132018 ms
-----------------------------------------------------
Suppressions used:
count bytes template
4 1192 libcrypto.so.1.1
-----------------------------------------------------
I20240502 14:16:09.158094 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.133:41841
--local_ip_for_outbound_sockets=127.27.220.133
--tserver_master_addrs=127.27.220.190:35583
--webserver_port=45569
--webserver_interface=127.27.220.133
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
W20240502 14:16:09.524384 4549 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:09.524755 4549 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:09.524947 4549 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:09.532807 4549 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:09.532958 4549 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:09.533131 4549 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.133
I20240502 14:16:09.541978 4549 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.133:41841
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.27.220.133
--webserver_port=45569
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.133
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:09.543570 4549 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:09.545614 4549 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:09.558977 4556 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:09.559712 4557 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:09.560442 4559 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:09.561218 4549 server_base.cc:1034] running on GCE node
I20240502 14:16:09.561828 4549 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:09.562762 4549 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:09.563962 4549 hybrid_clock.cc:648] HybridClock initialized: now 1714659369563903 us; error 48 us; skew 500 ppm
I20240502 14:16:09.564271 4549 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:09.567088 4549 webserver.cc:460] Webserver started at http://127.27.220.133:45569/ using document root <none> and password file <none>
I20240502 14:16:09.567775 4549 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:09.567932 4549 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:09.573562 4549 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.000s sys 0.003s
I20240502 14:16:09.576391 4566 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:09.577788 4549 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:16:09.577941 4549 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4"
format_stamp: "Formatted at 2024-05-02 14:15:51 on dist-test-slave-j72s"
I20240502 14:16:09.578734 4549 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:09.615041 4549 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:09.615921 4549 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:09.616257 4549 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:09.617254 4549 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:09.618844 4549 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:09.618916 4549 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:09.619074 4549 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:09.619131 4549 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:09.651914 4549 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.133:41841
I20240502 14:16:09.651966 4678 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.133:41841 every 8 connection(s)
I20240502 14:16:09.653429 4549 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 14:16:09.661047 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 4549
I20240502 14:16:09.661815 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.134:41403
--local_ip_for_outbound_sockets=127.27.220.134
--tserver_master_addrs=127.27.220.190:35583
--webserver_port=44617
--webserver_interface=127.27.220.134
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:16:09.666524 4679 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:16:09.666893 4679 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:09.667894 4679 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:16:09.669447 2577 ts_manager.cc:194] Re-registered known tserver with Master: 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
I20240502 14:16:09.670380 2577 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.133:59903
W20240502 14:16:10.009644 4684 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:10.009897 4684 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:10.010038 4684 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:10.017865 4684 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:10.017980 4684 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:10.018141 4684 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.134
I20240502 14:16:10.028339 4684 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.134:41403
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.27.220.134
--webserver_port=44617
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.134
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:10.029742 4684 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:10.031306 4684 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:10.044540 4689 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:10.045254 4692 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:10.045848 4690 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:10.046626 4684 server_base.cc:1034] running on GCE node
I20240502 14:16:10.047263 4684 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:10.048266 4684 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:10.049463 4684 hybrid_clock.cc:648] HybridClock initialized: now 1714659370049418 us; error 33 us; skew 500 ppm
I20240502 14:16:10.049790 4684 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:10.052417 4684 webserver.cc:460] Webserver started at http://127.27.220.134:44617/ using document root <none> and password file <none>
I20240502 14:16:10.053143 4684 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:10.053274 4684 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:10.058785 4684 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.000s sys 0.004s
I20240502 14:16:10.061426 4698 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:10.062690 4684 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 14:16:10.062837 4684 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "763dcf9158e648b496b4faa1a0dc54af"
format_stamp: "Formatted at 2024-05-02 14:15:51 on dist-test-slave-j72s"
I20240502 14:16:10.063649 4684 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:10.082289 4684 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:10.083266 4684 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:10.083586 4684 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:10.084671 4684 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:10.086618 4684 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:10.086714 4684 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:10.086839 4684 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:10.086902 4684 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:10.123749 4684 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.134:41403
I20240502 14:16:10.123790 4810 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.134:41403 every 8 connection(s)
I20240502 14:16:10.125206 4684 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 14:16:10.126942 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 4684
I20240502 14:16:10.127646 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.135:35331
--local_ip_for_outbound_sockets=127.27.220.135
--tserver_master_addrs=127.27.220.190:35583
--webserver_port=46739
--webserver_interface=127.27.220.135
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:16:10.139148 4811 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:16:10.139479 4811 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:10.140383 4811 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:16:10.141943 2578 ts_manager.cc:194] Re-registered known tserver with Master: 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403)
I20240502 14:16:10.143263 2578 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.134:44445
W20240502 14:16:10.487586 4814 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:10.487919 4814 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:10.488082 4814 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:10.496832 4814 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:10.497071 4814 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:10.497327 4814 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.135
I20240502 14:16:10.507014 4814 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.135:35331
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.27.220.135
--webserver_port=46739
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.135
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:10.508383 4814 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:10.510139 4814 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:10.522847 4821 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:10.523381 4824 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:10.524202 4822 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:10.524967 4814 server_base.cc:1034] running on GCE node
I20240502 14:16:10.525692 4814 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:10.526708 4814 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:10.527911 4814 hybrid_clock.cc:648] HybridClock initialized: now 1714659370527869 us; error 32 us; skew 500 ppm
I20240502 14:16:10.528229 4814 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:10.530552 4814 webserver.cc:460] Webserver started at http://127.27.220.135:46739/ using document root <none> and password file <none>
I20240502 14:16:10.531297 4814 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:10.531461 4814 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:10.536877 4814 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.004s sys 0.001s
I20240502 14:16:10.539732 4830 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:10.541179 4814 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:16:10.541342 4814 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "21e5a60e02fd475bb47e5b3a2f294333"
format_stamp: "Formatted at 2024-05-02 14:15:52 on dist-test-slave-j72s"
I20240502 14:16:10.542388 4814 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:10.561133 4814 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:10.562067 4814 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:10.562379 4814 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:10.563596 4814 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:10.565213 4814 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:10.565304 4814 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:10.565443 4814 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:10.565493 4814 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:10.596824 4814 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.135:35331
I20240502 14:16:10.596864 4942 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.135:35331 every 8 connection(s)
I20240502 14:16:10.598332 4814 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 14:16:10.603016 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 4814
I20240502 14:16:10.603600 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.136:39149
--local_ip_for_outbound_sockets=127.27.220.136
--tserver_master_addrs=127.27.220.190:35583
--webserver_port=41141
--webserver_interface=127.27.220.136
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:16:10.611560 4943 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:16:10.611883 4943 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:10.612851 4943 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:16:10.614738 2578 ts_manager.cc:194] Re-registered known tserver with Master: 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331)
I20240502 14:16:10.615621 2578 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.135:56019
I20240502 14:16:10.673174 4679 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
W20240502 14:16:10.962620 4946 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:10.963025 4946 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:10.963196 4946 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:10.971215 4946 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:10.971343 4946 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:10.971477 4946 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.136
I20240502 14:16:10.980938 4946 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.136:39149
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.27.220.136
--webserver_port=41141
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.136
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:10.982393 4946 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:10.984464 4946 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:10.997138 4953 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:10.998239 4952 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:10.998585 4955 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:10.999320 4946 server_base.cc:1034] running on GCE node
I20240502 14:16:10.999908 4946 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:11.000828 4946 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:11.002032 4946 hybrid_clock.cc:648] HybridClock initialized: now 1714659371001973 us; error 46 us; skew 500 ppm
I20240502 14:16:11.002375 4946 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:11.004679 4946 webserver.cc:460] Webserver started at http://127.27.220.136:41141/ using document root <none> and password file <none>
I20240502 14:16:11.005452 4946 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:11.005628 4946 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:11.010931 4946 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.001s sys 0.003s
I20240502 14:16:11.014788 4961 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:11.016387 4946 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.004s sys 0.000s
I20240502 14:16:11.016534 4946 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "fa04f9da159541bfbbb6ac55db06e07b"
format_stamp: "Formatted at 2024-05-02 14:15:52 on dist-test-slave-j72s"
I20240502 14:16:11.017351 4946 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:11.044437 4946 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:11.045481 4946 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:11.045866 4946 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:11.047036 4946 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:11.049278 4946 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:11.049405 4946 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
I20240502 14:16:11.049598 4946 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:11.049669 4946 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:11.108448 4946 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.136:39149
I20240502 14:16:11.108501 5074 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.136:39149 every 8 connection(s)
I20240502 14:16:11.109968 4946 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 14:16:11.118343 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 4946
I20240502 14:16:11.125562 5075 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:16:11.125902 5075 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:11.126767 5075 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:16:11.128219 2578 ts_manager.cc:194] Re-registered known tserver with Master: fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149)
I20240502 14:16:11.129190 2578 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.136:41935
I20240502 14:16:11.146999 4811 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
W20240502 14:16:11.494153 5080 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:11.494416 5080 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:11.502879 5080 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:16:11.618148 4943 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
I20240502 14:16:11.824156 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:11.824364 4878 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:11.835400 3091 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:11.838716 3091 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:11.839226 2701 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 14:16:11.843804 4878 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:11.870707 4614 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:11.876345 4614 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:11.895736 2963 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:11.922788 2701 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 14:16:11.928673 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:11.946643 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:11.963374 2963 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:11.973734 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:12.039703 5009 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:12.061065 5009 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:12.132515 5075 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
I20240502 14:16:12.351321 4878 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:12.361990 3091 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:12.369575 2701 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 14:16:12.376186 4614 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:12.439195 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:12.442637 2963 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:12.445777 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:12.473022 5009 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:12.737919 4878 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:12.755241 4614 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:12.767465 2701 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 14:16:12.794422 3091 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:12.798993 2963 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:12.809685 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:12.834447 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 14:16:12.924249 5214 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:12.924772 5214 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:16:12.935873 5009 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 14:16:12.938226 5214 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:16:13.038316 2721 consensus_queue.cc:237] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:13.042132 2983 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.043676 4815 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.044505 2852 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.045162 4815 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.050047 4283 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.055516 2852 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
W20240502 14:16:13.056576 2656 consensus_peers.cc:485] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 81be921c89194d3bb9c3d41be938ed74. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.065143 2579 catalog_manager.cc:5484] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.067003 2983 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.094228 3113 consensus_queue.cc:237] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:13.112412 2852 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.113184 4816 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.120062 4816 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.130641 2852 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
W20240502 14:16:13.139822 3047 consensus_peers.cc:485] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7f7589fa40dd4af09cc87e6e793bcd8a. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.142737 2572 catalog_manager.cc:5484] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.183791 2983 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 14:16:13.184669 3049 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:44843, user_credentials={real_user=slave}} blocked reactor thread for 44118.1us
I20240502 14:16:13.190833 5316 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.192932 2983 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.231249 2721 consensus_queue.cc:237] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:13.234347 4878 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:13.234973 3091 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:13.249511 2983 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.250595 5294 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.265723 4614 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:13.269466 3111 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.270284 5294 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.278658 5305 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
W20240502 14:16:13.281558 2656 consensus_peers.cc:485] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e882402ebdb24b03a9c51d74dcfd3550. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.296747 2701 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 14:16:13.298646 2572 catalog_manager.cc:5484] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:13.291721 2983 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.322655 3113 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.360934 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:13.399142 2983 consensus_queue.cc:237] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:13.421978 3111 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.422057 2721 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.422775 4529 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.424593 5354 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.425941 5009 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:13.426208 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
W20240502 14:16:13.451030 2916 consensus_peers.cc:485] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a1a32a28425e442ca4846e423e457345. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.452217 2963 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:13.455267 5354 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.458346 3113 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.463151 2721 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.472270 2572 catalog_manager.cc:5484] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:13.507580 2983 consensus_queue.cc:237] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:13.510411 2721 raft_consensus.cc:1261] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.510704 2852 raft_consensus.cc:1261] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.511003 4529 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.511569 4529 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.514549 5354 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.515134 2852 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.515147 2721 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
W20240502 14:16:13.517913 2917 consensus_peers.cc:485] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b9b5ec9436264dd9bd1f4e51bb1a5fb0. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.520470 2572 catalog_manager.cc:5484] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:13.538213 3113 consensus_queue.cc:237] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
W20240502 14:16:13.540661 3047 consensus_peers.cc:485] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9fd6780b78f849198bc91ed9ae2d08be. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.541270 2852 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.541277 2983 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.541819 4816 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.542084 5316 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
Runtime error: ksck discovered errors
I20240502 14:16:13.554785 5316 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.557268 2852 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.558426 2983 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.564886 2579 catalog_manager.cc:5484] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.574483 5380 ts_tablet_manager.cc:902] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:13.581218 2721 consensus_queue.cc:237] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:13.584427 3113 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.584427 2851 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 14:16:13.584787 2656 consensus_peers.cc:485] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cdaeb3ec3d344b4d9ac520f3d4c6d771. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.584990 5305 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.585669 4283 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.594978 5305 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.596452 2851 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.603554 2572 catalog_manager.cc:5484] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:13.613476 5387 ts_tablet_manager.cc:902] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:13.613359 3113 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.630244 5380 tablet_copy_client.cc:308] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:13.643602 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 7f7589fa40dd4af09cc87e6e793bcd8a from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:32937)
I20240502 14:16:13.643905 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 7f7589fa40dd4af09cc87e6e793bcd8a from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:32937: session id = fa04f9da159541bfbbb6ac55db06e07b-7f7589fa40dd4af09cc87e6e793bcd8a
I20240502 14:16:13.648068 3133 tablet_copy_source_session.cc:205] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:13.653256 5380 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7f7589fa40dd4af09cc87e6e793bcd8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:13.658541 5387 tablet_copy_client.cc:308] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:13.659936 3113 consensus_queue.cc:237] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:13.671612 5380 tablet_copy_client.cc:783] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:13.672206 5380 tablet_copy_client.cc:647] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:13.674902 5390 tablet_copy_client.cc:1002] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:13.676411 2983 raft_consensus.cc:1261] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.677023 5324 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.680521 5324 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.682543 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 81be921c89194d3bb9c3d41be938ed74 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:60189)
I20240502 14:16:13.682611 2721 raft_consensus.cc:1261] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.682843 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 81be921c89194d3bb9c3d41be938ed74 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:60189: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-81be921c89194d3bb9c3d41be938ed74
I20240502 14:16:13.683326 5319 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.687427 2983 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.693966 2742 tablet_copy_source_session.cc:205] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:13.694125 2579 catalog_manager.cc:5484] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.700575 5387 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81be921c89194d3bb9c3d41be938ed74. 1 dirs total, 0 dirs full, 0 dirs failed
W20240502 14:16:13.701017 3047 consensus_peers.cc:485] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8dcd1e1699c8448692fff7c1ed36312a. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.705742 2721 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.717882 5387 tablet_copy_client.cc:783] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:13.724727 5410 ts_tablet_manager.cc:902] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:13.727007 2983 consensus_queue.cc:237] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:13.733816 5387 tablet_copy_client.cc:647] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:13.736812 5399 tablet_copy_client.cc:1002] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
W20240502 14:16:13.741034 2916 consensus_peers.cc:485] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 02adec8c6c704326afbedb70578df917. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.741243 3113 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.741691 2851 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.742178 5082 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.742640 5354 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.755446 5354 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.760583 2852 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.770138 2572 catalog_manager.cc:5484] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.774909 3113 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.776528 5410 tablet_copy_client.cc:308] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:13.802368 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet e882402ebdb24b03a9c51d74dcfd3550 from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:51615)
I20240502 14:16:13.802577 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet e882402ebdb24b03a9c51d74dcfd3550 from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:51615: session id = 21e5a60e02fd475bb47e5b3a2f294333-e882402ebdb24b03a9c51d74dcfd3550
I20240502 14:16:13.805656 2742 tablet_copy_source_session.cc:205] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:13.810328 5410 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e882402ebdb24b03a9c51d74dcfd3550. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:13.827157 5410 tablet_copy_client.cc:783] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:13.827481 5410 tablet_copy_client.cc:647] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:13.829452 5418 tablet_copy_client.cc:1002] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:13.830926 2851 consensus_queue.cc:237] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:13.834463 2721 raft_consensus.cc:1261] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.834473 2983 raft_consensus.cc:1261] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.835083 4482 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.835570 5426 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 14:16:13.843076 2787 consensus_peers.cc:485] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3301dc9fca444af68c8a55dded735b38. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.844043 4482 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.845856 2983 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.849563 2721 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.854261 2579 catalog_manager.cc:5484] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:13.876466 2721 consensus_queue.cc:237] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:13.878934 3113 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.880056 5294 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.880218 2983 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.883193 5305 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 14:16:13.890039 2655 consensus_peers.cc:485] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4aa8dd00e4bb46e2bd8ec2888a54d46b. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.891832 5305 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.896597 2982 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.896418 3113 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.903862 2572 catalog_manager.cc:5484] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:13.924394 2721 consensus_queue.cc:237] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
W20240502 14:16:13.927073 2656 consensus_peers.cc:485] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e46c5705a2aa4a3fbb410a284e5ea228. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.927170 2851 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.927670 3113 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.927764 5442 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.928267 5305 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.931077 5442 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.931691 2851 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.932116 3113 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.937757 2579 catalog_manager.cc:5484] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:13.951695 5456 ts_tablet_manager.cc:902] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:13.954389 2721 consensus_queue.cc:237] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
W20240502 14:16:13.957893 2656 consensus_peers.cc:485] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 652af8d083784c078c5b45cca32ece4b. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:13.959278 2851 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.959513 3113 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:13.960294 5442 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.960747 5294 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:13.964699 5442 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.970484 3113 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.976089 2851 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.981202 2572 catalog_manager.cc:5484] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:13.981976 5456 tablet_copy_client.cc:308] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:14.003351 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet a1a32a28425e442ca4846e423e457345 from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:55427)
I20240502 14:16:14.003577 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet a1a32a28425e442ca4846e423e457345 from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:55427: session id = 763dcf9158e648b496b4faa1a0dc54af-a1a32a28425e442ca4846e423e457345
I20240502 14:16:14.007131 3003 tablet_copy_source_session.cc:205] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.006973 3113 consensus_queue.cc:237] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:14.011440 2721 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.012665 5316 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:14.014029 5456 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a1a32a28425e442ca4846e423e457345. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.014335 2982 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.018821 5324 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 14:16:14.026458 3046 consensus_peers.cc:485] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 609b87df71e64919a29a4fa093dfca23. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:14.029628 5473 ts_tablet_manager.cc:902] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:14.030087 5316 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:14.032371 5456 tablet_copy_client.cc:783] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.032817 5456 tablet_copy_client.cc:647] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.035423 2982 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:14.035874 5463 tablet_copy_client.cc:1002] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.037528 5473 tablet_copy_client.cc:308] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:14.040555 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:51615)
I20240502 14:16:14.040777 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:51615: session id = 21e5a60e02fd475bb47e5b3a2f294333-cdaeb3ec3d344b4d9ac520f3d4c6d771
I20240502 14:16:14.040573 2721 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:14.045409 2572 catalog_manager.cc:5484] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:14.055009 2742 tablet_copy_source_session.cc:205] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.056313 5473 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.061472 5473 tablet_copy_client.cc:783] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.061782 5473 tablet_copy_client.cc:647] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.063833 5477 tablet_copy_client.cc:1002] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.071110 5483 ts_tablet_manager.cc:902] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:14.073714 3113 consensus_queue.cc:237] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:14.076300 2721 raft_consensus.cc:1261] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.076496 2851 raft_consensus.cc:1261] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.076752 4816 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 14:16:14.077558 3047 consensus_peers.cc:485] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 88148a10290c4c48b87ab7e6aae5694c. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:14.077718 4816 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:14.083823 4816 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:14.089303 2851 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:14.099058 2579 catalog_manager.cc:5484] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:14.124910 5490 ts_tablet_manager.cc:902] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:14.143497 2722 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:14.157239 3113 consensus_queue.cc:237] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:14.159660 2722 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.159740 2982 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 14:16:14.159852 3047 consensus_peers.cc:485] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4f3030ae5dc2425586db21b6667e2deb. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:14.160295 5324 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:14.160578 5319 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:14.170096 5324 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:14.172688 5498 ts_tablet_manager.cc:902] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:14.174113 2982 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:14.175189 2722 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:14.178460 2577 catalog_manager.cc:5484] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:14.189935 5483 tablet_copy_client.cc:308] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:14.191218 5490 tablet_copy_client.cc:308] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:14.192222 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 9fd6780b78f849198bc91ed9ae2d08be from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:32937)
I20240502 14:16:14.192476 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 9fd6780b78f849198bc91ed9ae2d08be from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:32937: session id = fa04f9da159541bfbbb6ac55db06e07b-9fd6780b78f849198bc91ed9ae2d08be
I20240502 14:16:14.197057 3133 tablet_copy_source_session.cc:205] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.198753 5490 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9fd6780b78f849198bc91ed9ae2d08be. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.205866 5490 tablet_copy_client.cc:783] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.206182 5490 tablet_copy_client.cc:647] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.207816 5506 tablet_copy_client.cc:1002] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.210026 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:49647)
I20240502 14:16:14.210194 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:49647: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-b9b5ec9436264dd9bd1f4e51bb1a5fb0
I20240502 14:16:14.212343 3003 tablet_copy_source_session.cc:205] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.212524 5498 tablet_copy_client.cc:308] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:14.212298 2722 consensus_queue.cc:237] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:14.213589 5483 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0. 1 dirs total, 0 dirs full, 0 dirs failed
W20240502 14:16:14.215077 2656 consensus_peers.cc:485] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: aa0b526e216f46f0a2fb95fe0be02c46. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:14.215163 2982 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.215204 2851 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.215864 5442 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:14.216188 5294 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:14.220263 5483 tablet_copy_client.cc:783] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.220489 5483 tablet_copy_client.cc:647] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.222752 5503 tablet_copy_client.cc:1002] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.228771 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 8dcd1e1699c8448692fff7c1ed36312a from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:57321)
I20240502 14:16:14.228919 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 8dcd1e1699c8448692fff7c1ed36312a from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:57321: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-8dcd1e1699c8448692fff7c1ed36312a
I20240502 14:16:14.230973 3133 tablet_copy_source_session.cc:205] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.232316 5498 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8dcd1e1699c8448692fff7c1ed36312a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.238229 4283 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:14.239130 2851 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:14.240028 5498 tablet_copy_client.cc:783] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.240262 5498 tablet_copy_client.cc:647] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.242422 2982 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:14.243310 5514 tablet_copy_client.cc:1002] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.245733 2579 catalog_manager.cc:5484] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:14.273763 3113 consensus_queue.cc:237] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:14.277077 2851 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.277117 2982 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.277799 5316 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:14.280005 5316 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:14.293973 5316 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:14.298559 2852 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
W20240502 14:16:14.307056 3047 consensus_peers.cc:485] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6cdd66f946db45478a93459ba69702b2. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:14.308540 2579 catalog_manager.cc:5484] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:14.325079 2982 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:14.339567 5530 ts_tablet_manager.cc:902] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:14.349952 5527 ts_tablet_manager.cc:902] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:14.360366 5527 tablet_copy_client.cc:308] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:44843
I20240502 14:16:14.362336 2851 consensus_queue.cc:237] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:14.367831 3113 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.368532 5432 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 14:16:14.377846 2786 consensus_peers.cc:485] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e9b6901a9a524fd2ab1adb9d730fcd18. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:14.379418 2722 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:14.380597 5548 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:14.385372 5547 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.001s
I20240502 14:16:14.386015 3113 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:14.392657 2572 catalog_manager.cc:5484] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:14.398362 2873 tablet_copy_service.cc:140] P c6abc8535fe24d03bf71fb7092072fc4: Received BeginTabletCopySession request for tablet 3301dc9fca444af68c8a55dded735b38 from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:41937)
I20240502 14:16:14.398612 2873 tablet_copy_service.cc:161] P c6abc8535fe24d03bf71fb7092072fc4: Beginning new tablet copy session on tablet 3301dc9fca444af68c8a55dded735b38 from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:41937: session id = fa04f9da159541bfbbb6ac55db06e07b-3301dc9fca444af68c8a55dded735b38
I20240502 14:16:14.400218 2722 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:14.404882 2873 tablet_copy_source_session.cc:205] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.410006 5527 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3301dc9fca444af68c8a55dded735b38. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.418722 5527 tablet_copy_client.cc:783] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
W20240502 14:16:14.418498 5417 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
I20240502 14:16:14.419091 5527 tablet_copy_client.cc:647] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
W20240502 14:16:14.419145 5417 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:16:14.420084 5530 tablet_copy_client.cc:308] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:14.420898 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet 02adec8c6c704326afbedb70578df917 from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:55427)
I20240502 14:16:14.421062 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet 02adec8c6c704326afbedb70578df917 from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:55427: session id = 763dcf9158e648b496b4faa1a0dc54af-02adec8c6c704326afbedb70578df917
I20240502 14:16:14.421231 5542 tablet_copy_client.cc:1002] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.424211 3003 tablet_copy_source_session.cc:205] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.425999 5530 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02adec8c6c704326afbedb70578df917. 1 dirs total, 0 dirs full, 0 dirs failed
W20240502 14:16:14.431164 5417 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:16:14.432785 5530 tablet_copy_client.cc:783] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.433094 5530 tablet_copy_client.cc:647] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.434931 5553 tablet_copy_client.cc:1002] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.448724 5559 ts_tablet_manager.cc:902] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:14.471583 5559 tablet_copy_client.cc:308] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:14.479879 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:35351)
I20240502 14:16:14.480094 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:35351: session id = 763dcf9158e648b496b4faa1a0dc54af-4aa8dd00e4bb46e2bd8ec2888a54d46b
I20240502 14:16:14.483103 2742 tablet_copy_source_session.cc:205] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.486173 5559 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.499269 5559 tablet_copy_client.cc:783] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.499715 5559 tablet_copy_client.cc:647] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.511466 5566 tablet_copy_client.cc:1002] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.515291 5578 ts_tablet_manager.cc:902] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:14.519754 5578 tablet_copy_client.cc:308] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:14.520406 5582 ts_tablet_manager.cc:902] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:14.520735 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 652af8d083784c078c5b45cca32ece4b from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:60189)
I20240502 14:16:14.520921 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 652af8d083784c078c5b45cca32ece4b from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:60189: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-652af8d083784c078c5b45cca32ece4b
I20240502 14:16:14.523301 2742 tablet_copy_source_session.cc:205] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.524575 5578 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 652af8d083784c078c5b45cca32ece4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.534644 5578 tablet_copy_client.cc:783] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.535063 5578 tablet_copy_client.cc:647] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.537045 5583 tablet_copy_client.cc:1002] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.550417 5582 tablet_copy_client.cc:308] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:14.551474 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet e46c5705a2aa4a3fbb410a284e5ea228 from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:51615)
I20240502 14:16:14.551667 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet e46c5705a2aa4a3fbb410a284e5ea228 from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:51615: session id = 21e5a60e02fd475bb47e5b3a2f294333-e46c5705a2aa4a3fbb410a284e5ea228
I20240502 14:16:14.553653 2742 tablet_copy_source_session.cc:205] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.554860 5582 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e46c5705a2aa4a3fbb410a284e5ea228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.560050 5582 tablet_copy_client.cc:783] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.560333 5582 tablet_copy_client.cc:647] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.563006 5589 tablet_copy_client.cc:1002] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.566385 5592 ts_tablet_manager.cc:902] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:14.589584 5592 tablet_copy_client.cc:308] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:14.599100 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 609b87df71e64919a29a4fa093dfca23 from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:42215)
I20240502 14:16:14.599328 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 609b87df71e64919a29a4fa093dfca23 from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:42215: session id = 763dcf9158e648b496b4faa1a0dc54af-609b87df71e64919a29a4fa093dfca23
I20240502 14:16:14.600629 5602 ts_tablet_manager.cc:902] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:14.602671 3133 tablet_copy_source_session.cc:205] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.604705 5592 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 609b87df71e64919a29a4fa093dfca23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.613929 5592 tablet_copy_client.cc:783] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.614339 5592 tablet_copy_client.cc:647] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.616020 5602 tablet_copy_client.cc:308] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:14.617208 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 4f3030ae5dc2425586db21b6667e2deb from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:57321)
I20240502 14:16:14.617385 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 4f3030ae5dc2425586db21b6667e2deb from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:57321: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-4f3030ae5dc2425586db21b6667e2deb
I20240502 14:16:14.618225 5601 tablet_copy_client.cc:1002] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.621387 3133 tablet_copy_source_session.cc:205] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.623167 5602 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f3030ae5dc2425586db21b6667e2deb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.634045 5602 tablet_copy_client.cc:783] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.634294 5602 tablet_copy_client.cc:647] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.635856 5609 tablet_copy_client.cc:1002] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.684336 5636 ts_tablet_manager.cc:902] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:14.698151 5636 tablet_copy_client.cc:308] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:14.698937 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 88148a10290c4c48b87ab7e6aae5694c from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:32937)
I20240502 14:16:14.699136 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 88148a10290c4c48b87ab7e6aae5694c from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:32937: session id = fa04f9da159541bfbbb6ac55db06e07b-88148a10290c4c48b87ab7e6aae5694c
I20240502 14:16:14.701769 3133 tablet_copy_source_session.cc:205] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.703436 5636 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 88148a10290c4c48b87ab7e6aae5694c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.712975 5636 tablet_copy_client.cc:783] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.713299 5636 tablet_copy_client.cc:647] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.715435 5647 tablet_copy_client.cc:1002] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.770512 5665 ts_tablet_manager.cc:902] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:14.805598 4878 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:14.806344 3091 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:14.814337 4614 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:14.833719 5665 tablet_copy_client.cc:308] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:14.834558 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet aa0b526e216f46f0a2fb95fe0be02c46 from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:51615)
I20240502 14:16:14.834743 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet aa0b526e216f46f0a2fb95fe0be02c46 from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:51615: session id = 21e5a60e02fd475bb47e5b3a2f294333-aa0b526e216f46f0a2fb95fe0be02c46
I20240502 14:16:14.838719 2742 tablet_copy_source_session.cc:205] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:14.840279 5665 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa0b526e216f46f0a2fb95fe0be02c46. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:14.864372 2701 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 14:16:14.876734 5665 tablet_copy_client.cc:783] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:14.877049 5665 tablet_copy_client.cc:647] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:14.879434 5680 tablet_copy_client.cc:1002] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:14.896145 2963 tablet_service.cc:1429] Tablet server has 8 leaders and 0 scanners
I20240502 14:16:14.905980 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:14.907701 5688 ts_tablet_manager.cc:902] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:14.918936 5009 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:14.926352 5693 ts_tablet_manager.cc:902] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:14.929268 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:15.012823 5688 tablet_copy_client.cc:308] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:15.016420 5693 tablet_copy_client.cc:308] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:44843
I20240502 14:16:15.023133 2873 tablet_copy_service.cc:140] P c6abc8535fe24d03bf71fb7092072fc4: Received BeginTabletCopySession request for tablet e9b6901a9a524fd2ab1adb9d730fcd18 from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:43777)
I20240502 14:16:15.023308 2873 tablet_copy_service.cc:161] P c6abc8535fe24d03bf71fb7092072fc4: Beginning new tablet copy session on tablet e9b6901a9a524fd2ab1adb9d730fcd18 from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:43777: session id = 763dcf9158e648b496b4faa1a0dc54af-e9b6901a9a524fd2ab1adb9d730fcd18
I20240502 14:16:15.026422 2873 tablet_copy_source_session.cc:205] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:15.029693 5693 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b6901a9a524fd2ab1adb9d730fcd18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:15.032466 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 6cdd66f946db45478a93459ba69702b2 from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:36283)
I20240502 14:16:15.032629 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 6cdd66f946db45478a93459ba69702b2 from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:36283: session id = 21e5a60e02fd475bb47e5b3a2f294333-6cdd66f946db45478a93459ba69702b2
I20240502 14:16:15.035512 3133 tablet_copy_source_session.cc:205] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:15.036801 5688 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cdd66f946db45478a93459ba69702b2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:15.037761 5693 tablet_copy_client.cc:783] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:15.038064 5693 tablet_copy_client.cc:647] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:15.043383 5688 tablet_copy_client.cc:783] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:15.043978 5688 tablet_copy_client.cc:647] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:15.045367 5704 tablet_copy_client.cc:1002] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:15.045768 5701 tablet_copy_client.cc:1002] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
Runtime error: ksck discovered errors
I20240502 14:16:15.175453 5380 tablet_copy_client.cc:523] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.180179 5380 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:15.193245 5380 log.cc:824] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:15.237390 5387 tablet_copy_client.cc:523] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.239861 5380 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.240439 5380 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:15.240917 5380 ts_tablet_manager.cc:1367] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.061s user 0.060s sys 0.000s
I20240502 14:16:15.241554 5387 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:15.247581 5380 raft_consensus.cc:348] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:15.248056 5380 raft_consensus.cc:729] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:15.248668 5380 consensus_queue.cc:260] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:15.250857 5380 ts_tablet_manager.cc:1398] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.010s user 0.011s sys 0.000s
I20240502 14:16:15.251935 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-7f7589fa40dd4af09cc87e6e793bcd8a received from {username='slave'} at 127.27.220.136:32937
I20240502 14:16:15.252139 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-7f7589fa40dd4af09cc87e6e793bcd8a on tablet 7f7589fa40dd4af09cc87e6e793bcd8a with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:15.267024 5387 log.cc:824] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:15.273592 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.137:0
--local_ip_for_outbound_sockets=127.27.220.137
--webserver_interface=127.27.220.137
--webserver_port=0
--tserver_master_addrs=127.27.220.190:35583
--builtin_ntp_servers=127.27.220.148:43555
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 14:16:15.311034 5387 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.311724 5387 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:15.312268 5387 ts_tablet_manager.cc:1367] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.071s user 0.047s sys 0.007s
I20240502 14:16:15.318656 5387 raft_consensus.cc:348] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:15.319154 5387 raft_consensus.cc:729] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:15.319716 5387 consensus_queue.cc:260] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:15.322064 5387 ts_tablet_manager.cc:1398] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.010s user 0.012s sys 0.001s
I20240502 14:16:15.322997 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-81be921c89194d3bb9c3d41be938ed74 received from {username='slave'} at 127.27.220.133:60189
I20240502 14:16:15.323192 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-81be921c89194d3bb9c3d41be938ed74 on tablet 81be921c89194d3bb9c3d41be938ed74 with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:15.330034 5410 tablet_copy_client.cc:523] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.334656 5410 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:15.347518 5410 log.cc:824] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:15.380815 5410 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.381538 5410 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:15.382056 5410 ts_tablet_manager.cc:1367] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.048s user 0.040s sys 0.007s
I20240502 14:16:15.387743 5410 raft_consensus.cc:348] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:15.388260 5410 raft_consensus.cc:729] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:15.388803 5410 consensus_queue.cc:260] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:15.390956 5410 ts_tablet_manager.cc:1398] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.009s user 0.002s sys 0.005s
I20240502 14:16:15.392683 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-e882402ebdb24b03a9c51d74dcfd3550 received from {username='slave'} at 127.27.220.135:51615
I20240502 14:16:15.392854 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-e882402ebdb24b03a9c51d74dcfd3550 on tablet e882402ebdb24b03a9c51d74dcfd3550 with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:15.536769 5456 tablet_copy_client.cc:523] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.546095 5456 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:15.557932 5456 log.cc:824] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:15.564385 5473 tablet_copy_client.cc:523] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.569950 5473 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:15.577070 4898 raft_consensus.cc:1203] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:15.630177 5456 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.630756 5456 tablet_bootstrap.cc:492] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:15.631297 5456 ts_tablet_manager.cc:1367] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.088s user 0.066s sys 0.011s
I20240502 14:16:15.640041 5456 raft_consensus.cc:348] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:15.640725 5456 raft_consensus.cc:729] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:15.641465 5456 consensus_queue.cc:260] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:15.643999 4633 raft_consensus.cc:1203] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:15.651453 5473 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.651957 5473 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:15.652295 5473 ts_tablet_manager.cc:1367] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.082s user 0.048s sys 0.009s
I20240502 14:16:15.665925 5473 raft_consensus.cc:348] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:15.669098 5473 raft_consensus.cc:729] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:15.669373 5473 consensus_queue.cc:260] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:15.675078 5473 ts_tablet_manager.cc:1398] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.010s user 0.002s sys 0.000s
I20240502 14:16:15.676218 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-cdaeb3ec3d344b4d9ac520f3d4c6d771 received from {username='slave'} at 127.27.220.135:51615
I20240502 14:16:15.676388 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-cdaeb3ec3d344b4d9ac520f3d4c6d771 on tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:15.676995 5456 ts_tablet_manager.cc:1398] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.046s user 0.031s sys 0.005s
I20240502 14:16:15.678062 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-a1a32a28425e442ca4846e423e457345 received from {username='slave'} at 127.27.220.134:55427
I20240502 14:16:15.678261 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-a1a32a28425e442ca4846e423e457345 on tablet a1a32a28425e442ca4846e423e457345 with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:15.708245 5490 tablet_copy_client.cc:523] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.711249 5490 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:15.723129 5483 tablet_copy_client.cc:523] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.726786 5483 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:15.728560 4898 raft_consensus.cc:1203] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:15.743813 5498 tablet_copy_client.cc:523] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.746845 5029 raft_consensus.cc:1203] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:15.748420 5498 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
W20240502 14:16:15.799234 5744 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:15.799706 5744 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:15.799940 5744 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:16:15.808979 5490 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.808960 5498 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.809412 5490 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:15.809412 5498 tablet_bootstrap.cc:492] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:15.809743 5498 ts_tablet_manager.cc:1367] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.061s user 0.044s sys 0.006s
I20240502 14:16:15.810801 5498 raft_consensus.cc:348] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:15.811074 5498 raft_consensus.cc:729] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:15.811290 5498 consensus_queue.cc:260] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:15.812078 5498 ts_tablet_manager.cc:1398] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:15.812878 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-8dcd1e1699c8448692fff7c1ed36312a received from {username='slave'} at 127.27.220.133:57321
I20240502 14:16:15.813055 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-8dcd1e1699c8448692fff7c1ed36312a on tablet 8dcd1e1699c8448692fff7c1ed36312a with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:15.814440 5490 ts_tablet_manager.cc:1367] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.103s user 0.037s sys 0.013s
I20240502 14:16:15.815582 5490 raft_consensus.cc:348] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:15.815860 5490 raft_consensus.cc:729] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:15.816423 5490 consensus_queue.cc:260] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
W20240502 14:16:15.817132 5744 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:15.817279 5744 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:15.817484 5744 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.137
I20240502 14:16:15.821959 5490 ts_tablet_manager.cc:1398] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.007s user 0.001s sys 0.001s
I20240502 14:16:15.822067 4633 raft_consensus.cc:1203] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:15.822767 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-9fd6780b78f849198bc91ed9ae2d08be received from {username='slave'} at 127.27.220.136:32937
I20240502 14:16:15.822923 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-9fd6780b78f849198bc91ed9ae2d08be on tablet 9fd6780b78f849198bc91ed9ae2d08be with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:15.826558 5483 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.826980 5483 tablet_bootstrap.cc:492] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:15.827265 5483 ts_tablet_manager.cc:1367] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.101s user 0.046s sys 0.003s
I20240502 14:16:15.828025 5483 raft_consensus.cc:348] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:15.828225 5483 raft_consensus.cc:729] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:15.828425 5483 consensus_queue.cc:260] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:15.828886 5483 ts_tablet_manager.cc:1398] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 14:16:15.829558 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-b9b5ec9436264dd9bd1f4e51bb1a5fb0 received from {username='slave'} at 127.27.220.133:49647
I20240502 14:16:15.829707 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-b9b5ec9436264dd9bd1f4e51bb1a5fb0 on tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:15.835641 5744 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:43555
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.137:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.27.220.137
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.27.220.190:35583
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.137
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:15.837255 5744 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:15.839582 5744 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:15.857338 5844 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:15.858533 5848 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:15.863648 5744 server_base.cc:1034] running on GCE node
W20240502 14:16:15.865855 5851 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:15.866672 5744 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:15.867697 5744 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:15.868988 5744 hybrid_clock.cc:648] HybridClock initialized: now 1714659375868932 us; error 35 us; skew 500 ppm
I20240502 14:16:15.869361 5744 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:15.872155 5744 webserver.cc:460] Webserver started at http://127.27.220.137:33383/ using document root <none> and password file <none>
I20240502 14:16:15.872864 5744 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:15.872974 5744 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:15.873296 5744 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:15.875998 5744 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/instance:
uuid: "6fb8f2ad13f24e81935b7df7faa291fe"
format_stamp: "Formatted at 2024-05-02 14:16:15 on dist-test-slave-j72s"
I20240502 14:16:15.876717 5744 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal/instance:
uuid: "6fb8f2ad13f24e81935b7df7faa291fe"
format_stamp: "Formatted at 2024-05-02 14:16:15 on dist-test-slave-j72s"
I20240502 14:16:15.881866 5744 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240502 14:16:15.885396 5862 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:15.886672 5744 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:16:15.886834 5744 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "6fb8f2ad13f24e81935b7df7faa291fe"
format_stamp: "Formatted at 2024-05-02 14:16:15 on dist-test-slave-j72s"
I20240502 14:16:15.887089 5744 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:15.909075 5744 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:15.909881 5744 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:15.910159 5744 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:15.911264 5744 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:15.912942 5744 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:15.913030 5744 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:15.913163 5744 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:15.913256 5744 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:15.921715 5527 tablet_copy_client.cc:523] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.925390 5527 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:15.935384 5530 tablet_copy_client.cc:523] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:15.940927 5530 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:15.970942 5527 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:15.971403 5527 tablet_bootstrap.cc:492] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:15.971705 5527 ts_tablet_manager.cc:1367] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.046s user 0.034s sys 0.008s
I20240502 14:16:15.972707 5527 raft_consensus.cc:348] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:15.973016 5527 raft_consensus.cc:729] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:15.973258 5527 consensus_queue.cc:260] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:15.977460 5527 ts_tablet_manager.cc:1398] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.006s user 0.004s sys 0.000s
I20240502 14:16:15.982857 2873 tablet_copy_service.cc:342] P c6abc8535fe24d03bf71fb7092072fc4: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-3301dc9fca444af68c8a55dded735b38 received from {username='slave'} at 127.27.220.136:41937
I20240502 14:16:15.983029 2873 tablet_copy_service.cc:433] P c6abc8535fe24d03bf71fb7092072fc4: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-3301dc9fca444af68c8a55dded735b38 on tablet 3301dc9fca444af68c8a55dded735b38 with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:16.011889 5559 tablet_copy_client.cc:523] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.014259 5744 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.137:45251
I20240502 14:16:16.014411 5976 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.137:45251 every 8 connection(s)
I20240502 14:16:16.015110 5559 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:16.015889 5744 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240502 14:16:16.023075 5530 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.023510 5530 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:16.023857 5530 ts_tablet_manager.cc:1367] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.083s user 0.070s sys 0.009s
I20240502 14:16:16.024693 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 5744
I20240502 14:16:16.024756 5530 raft_consensus.cc:348] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:16.024961 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal/instance
I20240502 14:16:16.025015 5530 raft_consensus.cc:729] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:16.025245 5530 consensus_queue.cc:260] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:16.028661 5530 ts_tablet_manager.cc:1398] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 14:16:16.029371 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-02adec8c6c704326afbedb70578df917 received from {username='slave'} at 127.27.220.134:55427
I20240502 14:16:16.029563 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-02adec8c6c704326afbedb70578df917 on tablet 02adec8c6c704326afbedb70578df917 with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:16.037482 5578 tablet_copy_client.cc:523] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.037542 5977 heartbeater.cc:347] Connected to a master server at 127.27.220.190:35583
I20240502 14:16:16.037866 5977 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:16.038920 5977 heartbeater.cc:511] Master 127.27.220.190:35583 requested a full tablet report, sending...
I20240502 14:16:16.040431 5578 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:16.040520 2579 ts_manager.cc:194] Registered new tserver with Master: 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251)
I20240502 14:16:16.041618 2579 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.137:36995
I20240502 14:16:16.060247 5559 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.060652 5559 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:16.060952 5559 ts_tablet_manager.cc:1367] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.046s user 0.030s sys 0.007s
I20240502 14:16:16.061910 5559 raft_consensus.cc:348] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:16.062237 5559 raft_consensus.cc:729] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:16.062475 5559 consensus_queue.cc:260] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:16.063202 5559 ts_tablet_manager.cc:1398] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.002s user 0.005s sys 0.001s
I20240502 14:16:16.063446 5582 tablet_copy_client.cc:523] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.064098 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-4aa8dd00e4bb46e2bd8ec2888a54d46b received from {username='slave'} at 127.27.220.134:35351
I20240502 14:16:16.064257 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-4aa8dd00e4bb46e2bd8ec2888a54d46b on tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:16.066957 5582 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:16.078177 5029 raft_consensus.cc:1203] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.086931 5578 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.087473 5578 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:16.087769 5578 ts_tablet_manager.cc:1367] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.047s user 0.040s sys 0.004s
I20240502 14:16:16.088618 5578 raft_consensus.cc:348] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:16.088827 5578 raft_consensus.cc:729] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:16.089030 5578 consensus_queue.cc:260] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:16.090111 5578 ts_tablet_manager.cc:1398] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:16.090767 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-652af8d083784c078c5b45cca32ece4b received from {username='slave'} at 127.27.220.133:60189
I20240502 14:16:16.090924 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-652af8d083784c078c5b45cca32ece4b on tablet 652af8d083784c078c5b45cca32ece4b with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:16.118638 5592 tablet_copy_client.cc:523] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.122937 5592 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:16.123087 5582 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.123524 5582 tablet_bootstrap.cc:492] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:16.123805 5582 ts_tablet_manager.cc:1367] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.057s user 0.047s sys 0.008s
I20240502 14:16:16.124672 5582 raft_consensus.cc:348] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:16.124936 5582 raft_consensus.cc:729] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:16.125185 5582 consensus_queue.cc:260] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:16.125751 5582 ts_tablet_manager.cc:1398] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:16:16.126552 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-e46c5705a2aa4a3fbb410a284e5ea228 received from {username='slave'} at 127.27.220.135:51615
I20240502 14:16:16.126711 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-e46c5705a2aa4a3fbb410a284e5ea228 on tablet e46c5705a2aa4a3fbb410a284e5ea228 with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:16.132311 4633 raft_consensus.cc:1203] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.136307 5602 tablet_copy_client.cc:523] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.138618 5442 raft_consensus.cc:1050] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:16.139679 5442 consensus_queue.cc:237] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:16.140395 5602 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:16.143180 5294 raft_consensus.cc:1050] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:16.144001 5294 consensus_queue.cc:237] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.147179 3111 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.147272 4898 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.147965 4283 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.148555 6001 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.156584 2851 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.158069 2982 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.159454 4633 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.160516 4283 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.161114 4283 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.161661 4283 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.165948 4766 raft_consensus.cc:1203] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.172386 2983 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.173092 5294 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.176533 5316 raft_consensus.cc:1050] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:16.177575 5316 consensus_queue.cc:237] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:16.181421 5294 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.182291 5442 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.190661 4766 raft_consensus.cc:1203] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.197839 2579 catalog_manager.cc:5484] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.200603 3111 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.206816 2982 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.211334 2851 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.214119 4898 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.215934 5636 tablet_copy_client.cc:523] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.223975 2572 catalog_manager.cc:5484] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.226897 5636 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:16.228648 5029 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.229439 5324 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.229876 2982 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.230454 5324 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.233156 2851 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.233790 5324 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.233908 4633 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.235713 5592 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.236196 5592 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:16.236500 5592 ts_tablet_manager.cc:1367] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.114s user 0.049s sys 0.000s
I20240502 14:16:16.245209 5602 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.237303 5592 raft_consensus.cc:348] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:16.245590 5592 raft_consensus.cc:729] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:16.245972 5592 consensus_queue.cc:260] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:16.247823 5592 ts_tablet_manager.cc:1398] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.011s user 0.003s sys 0.000s
I20240502 14:16:16.248740 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-609b87df71e64919a29a4fa093dfca23 received from {username='slave'} at 127.27.220.134:42215
I20240502 14:16:16.248960 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-609b87df71e64919a29a4fa093dfca23 on tablet 609b87df71e64919a29a4fa093dfca23 with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:16.251070 5602 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:16.251533 5602 ts_tablet_manager.cc:1367] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.111s user 0.050s sys 0.003s
I20240502 14:16:16.252434 5602 raft_consensus.cc:348] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:16.252730 5602 raft_consensus.cc:729] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:16.252960 5602 consensus_queue.cc:260] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:16.253600 5602 ts_tablet_manager.cc:1398] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:16:16.255811 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-4f3030ae5dc2425586db21b6667e2deb received from {username='slave'} at 127.27.220.133:57321
I20240502 14:16:16.255986 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-4f3030ae5dc2425586db21b6667e2deb on tablet 4f3030ae5dc2425586db21b6667e2deb with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:16.256673 5316 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.257632 5029 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.262533 2852 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.255038 2983 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.264392 2577 catalog_manager.cc:5484] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b reported cstate change: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.276687 4633 raft_consensus.cc:1203] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.288364 2722 consensus_queue.cc:237] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.288606 2721 consensus_queue.cc:237] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:16.291678 4633 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.292162 2982 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.291718 2981 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.293123 2983 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.293222 4898 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.294358 4283 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.294598 3111 consensus_queue.cc:237] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:16.294961 4283 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.297212 5029 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.297776 5316 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.298156 2851 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.298780 5316 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.300779 5316 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.302474 5294 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.303071 5294 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.304328 4283 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.306265 2851 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.306963 5029 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.308768 4898 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.309326 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7f7589fa40dd4af09cc87e6e793bcd8a with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.311502 2576 catalog_manager.cc:5484] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.313112 6001 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.317871 4633 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.320567 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 7f7589fa40dd4af09cc87e6e793bcd8a with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:16.322090 6027 tablet_replica.cc:330] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:16.322575 2982 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.324649 2983 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.327081 6027 raft_consensus.cc:2229] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.327536 6027 raft_consensus.cc:2258] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.329382 6027 ts_tablet_manager.cc:1875] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.331285 5294 raft_consensus.cc:1050] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:16.332176 5294 consensus_queue.cc:237] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:16.333160 2577 catalog_manager.cc:5484] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.335392 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 81be921c89194d3bb9c3d41be938ed74 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.338125 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e882402ebdb24b03a9c51d74dcfd3550 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.338338 6027 ts_tablet_manager.cc:1888] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.338498 6027 log.cc:1196] T 7f7589fa40dd4af09cc87e6e793bcd8a P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/7f7589fa40dd4af09cc87e6e793bcd8a
I20240502 14:16:16.339457 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 7f7589fa40dd4af09cc87e6e793bcd8a (table rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]) successfully deleted
I20240502 14:16:16.340512 2578 catalog_manager.cc:5484] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.345147 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 81be921c89194d3bb9c3d41be938ed74 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:16.347857 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet e882402ebdb24b03a9c51d74dcfd3550 with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:16.350891 6030 tablet_replica.cc:330] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:16.351922 6030 raft_consensus.cc:2229] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.352583 6030 raft_consensus.cc:2258] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.355008 6030 ts_tablet_manager.cc:1875] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.359542 6029 tablet_replica.cc:330] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:16.360169 6029 raft_consensus.cc:2229] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.360693 6029 raft_consensus.cc:2258] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.362521 6029 ts_tablet_manager.cc:1875] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.362911 6030 ts_tablet_manager.cc:1888] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.363137 6030 log.cc:1196] T e882402ebdb24b03a9c51d74dcfd3550 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/e882402ebdb24b03a9c51d74dcfd3550
I20240502 14:16:16.364228 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet e882402ebdb24b03a9c51d74dcfd3550 (table rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]) successfully deleted
I20240502 14:16:16.366350 5636 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.368472 6029 ts_tablet_manager.cc:1888] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.368746 6029 log.cc:1196] T 81be921c89194d3bb9c3d41be938ed74 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/81be921c89194d3bb9c3d41be938ed74
I20240502 14:16:16.369843 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 81be921c89194d3bb9c3d41be938ed74 (table rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]) successfully deleted
I20240502 14:16:16.372335 5636 tablet_bootstrap.cc:492] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:16.372661 5636 ts_tablet_manager.cc:1367] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.146s user 0.042s sys 0.008s
I20240502 14:16:16.374367 2851 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.374400 4898 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.375110 6004 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.375957 3111 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.377760 6004 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.379870 5665 tablet_copy_client.cc:523] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.384553 5665 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:16.391906 5294 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.398074 5636 raft_consensus.cc:348] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:16.398519 5636 raft_consensus.cc:729] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:16.398803 5636 consensus_queue.cc:260] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:16.402891 5636 ts_tablet_manager.cc:1398] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.030s user 0.002s sys 0.000s
I20240502 14:16:16.405978 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-88148a10290c4c48b87ab7e6aae5694c received from {username='slave'} at 127.27.220.136:32937
I20240502 14:16:16.406152 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-88148a10290c4c48b87ab7e6aae5694c on tablet 88148a10290c4c48b87ab7e6aae5694c with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:16.407339 5029 raft_consensus.cc:1203] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.412158 5294 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.413888 4766 raft_consensus.cc:1203] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.417289 3113 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.421362 2851 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.421494 2575 catalog_manager.cc:5484] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.423157 4898 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.430553 5324 raft_consensus.cc:1050] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:16.431406 5324 consensus_queue.cc:237] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.434767 2721 consensus_queue.cc:237] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:16.437086 4897 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.438110 6001 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.439656 2721 raft_consensus.cc:1261] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.439707 4633 raft_consensus.cc:1261] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.440582 5324 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.440733 2983 raft_consensus.cc:1261] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.441177 5324 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.441488 5319 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.444135 4283 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.446501 4897 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.450119 4898 raft_consensus.cc:1203] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.451539 2851 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.452381 6001 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.454182 2852 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:16.456743 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.461701 2575 catalog_manager.cc:5484] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.467373 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:16.467885 6030 tablet_replica.cc:330] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:16.468129 6030 raft_consensus.cc:2229] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.468376 6030 raft_consensus.cc:2258] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.469491 5316 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.469491 6030 ts_tablet_manager.cc:1875] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.472728 4633 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.473492 2722 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.477087 6030 ts_tablet_manager.cc:1888] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.477265 6030 log.cc:1196] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/cdaeb3ec3d344b4d9ac520f3d4c6d771
I20240502 14:16:16.478137 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 (table rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]) successfully deleted
I20240502 14:16:16.479244 2575 catalog_manager.cc:5484] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.481055 2983 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.482724 5324 raft_consensus.cc:1050] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:16.483780 5324 consensus_queue.cc:237] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:16.484647 5665 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.486104 4633 raft_consensus.cc:1203] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.487545 5665 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:16.489547 5665 ts_tablet_manager.cc:1367] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.105s user 0.051s sys 0.011s
I20240502 14:16:16.490451 5665 raft_consensus.cc:348] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:16.490655 5665 raft_consensus.cc:729] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:16.490866 5665 consensus_queue.cc:260] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:16.492877 5665 ts_tablet_manager.cc:1398] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:16.493701 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-aa0b526e216f46f0a2fb95fe0be02c46 received from {username='slave'} at 127.27.220.135:51615
I20240502 14:16:16.493865 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-aa0b526e216f46f0a2fb95fe0be02c46 on tablet aa0b526e216f46f0a2fb95fe0be02c46 with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:16.494781 5029 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.496006 6048 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.497426 2851 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.498322 6048 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.500049 3113 consensus_queue.cc:237] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.502393 4633 raft_consensus.cc:1261] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.503016 6049 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.503536 2983 raft_consensus.cc:1261] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.504351 6049 consensus_queue.cc:1035] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.505669 6048 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.506255 4633 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.509433 4897 raft_consensus.cc:1203] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.509366 2983 raft_consensus.cc:2932] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.513267 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8dcd1e1699c8448692fff7c1ed36312a with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.515620 2983 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.515863 2577 catalog_manager.cc:5484] T 8dcd1e1699c8448692fff7c1ed36312a P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.517809 5324 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.521576 5324 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.526286 2851 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.528621 5029 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.529958 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet 8dcd1e1699c8448692fff7c1ed36312a with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:16.533051 2576 catalog_manager.cc:5484] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.538344 3113 consensus_queue.cc:237] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:16.540973 5029 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.541533 5324 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.542672 6060 tablet_replica.cc:330] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:16.543104 2852 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.544476 6048 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.544298 5319 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.545083 5029 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.545828 5693 tablet_copy_client.cc:523] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.546168 5688 tablet_copy_client.cc:523] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:16.547926 2851 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.549396 5693 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:16.551012 5688 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:16.552820 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9fd6780b78f849198bc91ed9ae2d08be with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.556103 2576 catalog_manager.cc:5484] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.562811 6060 raft_consensus.cc:2229] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.563203 6060 raft_consensus.cc:2258] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.565228 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 9fd6780b78f849198bc91ed9ae2d08be with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:16.565613 6027 tablet_replica.cc:330] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:16.565824 6027 raft_consensus.cc:2229] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.566120 6027 pending_rounds.cc:62] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: Trying to abort 1 pending ops.
I20240502 14:16:16.566224 6027 pending_rounds.cc:68] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7023244806074863616 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "9fd6780b78f849198bc91ed9ae2d08be" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } } }
I20240502 14:16:16.566689 6027 raft_consensus.cc:2894] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240502 14:16:16.566810 6027 raft_consensus.cc:2866] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 14:16:16.566936 6027 raft_consensus.cc:2258] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.567869 6027 ts_tablet_manager.cc:1875] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.568578 6060 ts_tablet_manager.cc:1875] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.574421 6027 ts_tablet_manager.cc:1888] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.574596 6027 log.cc:1196] T 9fd6780b78f849198bc91ed9ae2d08be P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/9fd6780b78f849198bc91ed9ae2d08be
I20240502 14:16:16.576020 6060 ts_tablet_manager.cc:1888] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.576215 6060 log.cc:1196] T 8dcd1e1699c8448692fff7c1ed36312a P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/8dcd1e1699c8448692fff7c1ed36312a
I20240502 14:16:16.576663 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 9fd6780b78f849198bc91ed9ae2d08be (table rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]) successfully deleted
I20240502 14:16:16.577226 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet 8dcd1e1699c8448692fff7c1ed36312a (table rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]) successfully deleted
I20240502 14:16:16.615633 5688 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.615676 5029 raft_consensus.cc:1203] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.616245 5688 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:16.616609 5688 ts_tablet_manager.cc:1367] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.066s user 0.047s sys 0.002s
I20240502 14:16:16.617492 5688 raft_consensus.cc:348] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:16.617883 5688 raft_consensus.cc:729] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:16.618155 5688 consensus_queue.cc:260] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:16.619529 5688 ts_tablet_manager.cc:1398] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:16.620155 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-6cdd66f946db45478a93459ba69702b2 received from {username='slave'} at 127.27.220.135:36283
I20240502 14:16:16.620313 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-6cdd66f946db45478a93459ba69702b2 on tablet 6cdd66f946db45478a93459ba69702b2 with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:16.623862 5693 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:16.624341 5693 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:16.624752 5693 ts_tablet_manager.cc:1367] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.075s user 0.053s sys 0.002s
I20240502 14:16:16.625782 5693 raft_consensus.cc:348] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:16.626093 5693 raft_consensus.cc:729] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:16.626427 5693 consensus_queue.cc:260] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:16.627214 5693 ts_tablet_manager.cc:1398] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:16.628398 2873 tablet_copy_service.cc:342] P c6abc8535fe24d03bf71fb7092072fc4: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-e9b6901a9a524fd2ab1adb9d730fcd18 received from {username='slave'} at 127.27.220.134:43777
I20240502 14:16:16.628528 2873 tablet_copy_service.cc:433] P c6abc8535fe24d03bf71fb7092072fc4: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-e9b6901a9a524fd2ab1adb9d730fcd18 on tablet e9b6901a9a524fd2ab1adb9d730fcd18 with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:16.630688 6075 raft_consensus.cc:1050] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:16.632643 6075 consensus_queue.cc:237] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:16.636507 2721 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.636984 4766 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.637121 6076 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20240502 14:16:16.639822 5573 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.645850 3113 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.648223 6076 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.656014 6075 raft_consensus.cc:1050] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:16.657025 6075 consensus_queue.cc:237] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.678280 5573 raft_consensus.cc:1050] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:16.679447 5573 consensus_queue.cc:237] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:16.689075 3111 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.689857 6076 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.692664 4765 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.693425 6076 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.705327 6076 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.708657 6082 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.708992 2721 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.710183 2851 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.712531 3113 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.715085 2575 catalog_manager.cc:5484] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.719172 4633 raft_consensus.cc:1261] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.719142 3111 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.721802 2851 raft_consensus.cc:1261] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.722770 6093 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20240502 14:16:16.724690 2576 catalog_manager.cc:5484] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.726436 4766 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.728758 6093 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.729363 6093 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20240502 14:16:16.730301 2721 raft_consensus.cc:1261] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.732755 4765 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.737620 6092 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.751039 4765 raft_consensus.cc:1203] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.755048 6083 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.755982 2721 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.756784 2853 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.761148 2982 consensus_queue.cc:237] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:16.761150 2983 consensus_queue.cc:237] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:16.765156 3113 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.765201 3111 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.765812 4765 raft_consensus.cc:1261] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.765986 6093 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.766527 6093 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.767022 6093 consensus_queue.cc:1035] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.767145 4765 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.769336 6082 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.769773 2577 catalog_manager.cc:5484] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.771312 6082 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.771932 3111 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.771956 6076 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.772804 4765 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.776808 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a1a32a28425e442ca4846e423e457345 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.779448 2578 catalog_manager.cc:5484] T a1a32a28425e442ca4846e423e457345 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.779453 2575 catalog_manager.cc:5484] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.781230 4766 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.784674 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet a1a32a28425e442ca4846e423e457345 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:16.784654 4633 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.785878 6060 tablet_replica.cc:330] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:16.786190 6060 raft_consensus.cc:2229] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.786669 6060 raft_consensus.cc:2258] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.787184 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 02adec8c6c704326afbedb70578df917 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:16.789016 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 02adec8c6c704326afbedb70578df917 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.789554 2851 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.792701 6060 ts_tablet_manager.cc:1875] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.797984 6029 tablet_replica.cc:330] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:16.798209 6029 raft_consensus.cc:2229] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.798445 6029 raft_consensus.cc:2258] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.799896 6029 ts_tablet_manager.cc:1875] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.803229 2983 consensus_queue.cc:237] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.803625 6007 raft_consensus.cc:1050] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:16.804476 6007 consensus_queue.cc:237] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.805929 4633 raft_consensus.cc:1261] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.806578 6082 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.807936 2721 raft_consensus.cc:1261] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.808482 6083 consensus_queue.cc:1035] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.810154 6083 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.811578 4633 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.812192 6001 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.812325 3111 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.813409 2851 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.813663 6001 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.813982 5294 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.815254 4634 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.816689 6060 ts_tablet_manager.cc:1888] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.818603 6060 log.cc:1196] T a1a32a28425e442ca4846e423e457345 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/a1a32a28425e442ca4846e423e457345
I20240502 14:16:16.818476 3110 raft_consensus.cc:2932] T a1a32a28425e442ca4846e423e457345 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.819471 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet a1a32a28425e442ca4846e423e457345 (table rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]) successfully deleted
I20240502 14:16:16.821048 4765 raft_consensus.cc:1203] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.821607 2722 raft_consensus.cc:2932] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.822908 2576 catalog_manager.cc:5484] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.823590 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.829360 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:16.831954 6029 ts_tablet_manager.cc:1888] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.832183 6029 log.cc:1196] T 02adec8c6c704326afbedb70578df917 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/02adec8c6c704326afbedb70578df917
I20240502 14:16:16.832964 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 02adec8c6c704326afbedb70578df917 (table rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]) successfully deleted
I20240502 14:16:16.833406 6029 tablet_replica.cc:330] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:16.833686 6029 raft_consensus.cc:2229] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.833999 6029 raft_consensus.cc:2258] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.835062 6029 ts_tablet_manager.cc:1875] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.841966 6029 ts_tablet_manager.cc:1888] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.842226 6029 log.cc:1196] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/b9b5ec9436264dd9bd1f4e51bb1a5fb0
I20240502 14:16:16.843243 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0 (table rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]) successfully deleted
I20240502 14:16:16.859508 5294 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.860146 2852 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.862296 4634 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.865010 3111 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.867767 2575 catalog_manager.cc:5484] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.873634 2722 consensus_queue.cc:237] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.875698 4634 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.876257 4283 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.878239 3111 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.878775 5294 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.879752 5294 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.880309 4634 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.883040 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 652af8d083784c078c5b45cca32ece4b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.884502 2578 catalog_manager.cc:5484] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.887981 3110 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.890681 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 652af8d083784c078c5b45cca32ece4b with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:16.891053 6029 tablet_replica.cc:330] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:16.891253 6029 raft_consensus.cc:2229] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.891562 6029 raft_consensus.cc:2258] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.894811 6029 ts_tablet_manager.cc:1875] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.895773 6001 raft_consensus.cc:1050] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:16.896831 6001 consensus_queue.cc:237] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:16.902900 6029 ts_tablet_manager.cc:1888] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.903049 6029 log.cc:1196] T 652af8d083784c078c5b45cca32ece4b P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/652af8d083784c078c5b45cca32ece4b
I20240502 14:16:16.904080 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 652af8d083784c078c5b45cca32ece4b (table rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]) successfully deleted
I20240502 14:16:16.906086 2983 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.907153 3110 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.907173 6001 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.907413 4765 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.908007 6004 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.908339 6007 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.913439 4765 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.913698 6004 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.918723 3110 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.920931 2577 catalog_manager.cc:5484] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:16.921875 5319 raft_consensus.cc:1050] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:16.921732 2983 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.922739 5319 consensus_queue.cc:237] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:16.928100 2722 consensus_queue.cc:237] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:16.932080 2983 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.933203 6004 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.936015 6007 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.941654 4897 raft_consensus.cc:1203] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:16.944876 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:16.945439 2983 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.947618 4765 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:16.947072 2578 catalog_manager.cc:5484] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:16.949906 6007 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:16.950387 2722 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.950421 4634 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.951140 6048 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.951745 6048 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.955632 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:16.955964 6030 tablet_replica.cc:330] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:16.956188 6030 raft_consensus.cc:2229] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:16.956545 6030 raft_consensus.cc:2258] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:16.957840 6030 ts_tablet_manager.cc:1875] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:16.957958 5552 raft_consensus.cc:1050] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:16.958992 5552 consensus_queue.cc:237] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:16.959272 4765 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:16.962656 2983 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.963249 5324 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.963514 2983 raft_consensus.cc:1261] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.964198 6100 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.964540 5029 raft_consensus.cc:1261] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.965216 6100 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.965328 5324 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.969676 2722 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.970402 6030 ts_tablet_manager.cc:1888] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:16.970974 6030 log.cc:1196] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/4aa8dd00e4bb46e2bd8ec2888a54d46b
I20240502 14:16:16.972222 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b (table rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]) successfully deleted
I20240502 14:16:16.978786 4634 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.983714 2578 catalog_manager.cc:5484] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.987861 6100 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.992569 2722 raft_consensus.cc:1261] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:16.992573 5029 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.993167 6135 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:16.994508 2982 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:16.996930 2722 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:16.999526 2983 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.001379 2576 catalog_manager.cc:5484] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:17.007685 6007 raft_consensus.cc:1050] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:17.008657 6007 consensus_queue.cc:237] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:17.014998 2852 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.015002 4897 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.015689 5294 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.015967 2982 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.016278 5294 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.016942 6004 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.032593 5294 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.035377 4897 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.039018 2982 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.039580 2852 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.042739 6007 raft_consensus.cc:1050] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:17.042562 2577 catalog_manager.cc:5484] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:17.043604 6007 consensus_queue.cc:237] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:17.045089 5977 heartbeater.cc:503] Master 127.27.220.190:35583 was elected leader, sending a full tablet report...
I20240502 14:16:17.051059 2722 consensus_queue.cc:237] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:17.052639 3110 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb"
dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:40106
I20240502 14:16:17.052945 3110 raft_consensus.cc:595] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:17.053318 4897 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.053730 2982 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.053901 6001 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.054466 6001 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.055045 4898 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.056188 2852 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.056774 6004 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.057358 4283 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.057701 5294 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.061300 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet aa0b526e216f46f0a2fb95fe0be02c46 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:17.062316 2982 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.063043 3110 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.062913 4897 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.063748 6147 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.067238 2576 catalog_manager.cc:5484] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:17.068739 6147 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.070885 3110 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.072579 2852 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.074980 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet aa0b526e216f46f0a2fb95fe0be02c46 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:17.075371 6029 tablet_replica.cc:330] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:17.075589 6029 raft_consensus.cc:2229] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:17.075892 6029 raft_consensus.cc:2258] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:17.078192 6029 ts_tablet_manager.cc:1875] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:17.078824 2575 catalog_manager.cc:5484] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:17.086557 4896 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.097409 6029 ts_tablet_manager.cc:1888] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:17.097609 6029 log.cc:1196] T aa0b526e216f46f0a2fb95fe0be02c46 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/aa0b526e216f46f0a2fb95fe0be02c46
I20240502 14:16:17.098469 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet aa0b526e216f46f0a2fb95fe0be02c46 (table rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]) successfully deleted
I20240502 14:16:17.181598 3110 consensus_queue.cc:237] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:17.184317 2722 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 14:16:17.184347 3047 consensus_peers.cc:485] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 28f1bc6021e04a05916d3e49a8c98597. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:17.184634 2852 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.184963 5319 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.185221 5324 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.187997 6048 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:17.188601 2852 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:17.191334 2722 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:17.195617 2576 catalog_manager.cc:5484] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:17.199821 6188 raft_consensus.cc:1050] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:17.200667 6188 consensus_queue.cc:237] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:17.203230 2852 raft_consensus.cc:1261] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.203814 5029 raft_consensus.cc:1261] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.204221 2722 raft_consensus.cc:1261] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.204784 5324 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.205327 5324 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.205665 5319 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.208469 5319 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.209112 2722 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.210566 5029 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.214654 2575 catalog_manager.cc:5484] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.214938 2982 consensus_queue.cc:237] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:17.214793 2852 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.221374 3111 consensus_queue.cc:237] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:17.223348 5029 raft_consensus.cc:1261] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.223644 2851 raft_consensus.cc:1261] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.224177 5324 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.224671 5319 consensus_queue.cc:1035] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.226979 2852 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.227648 6083 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.227604 6048 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.228112 3111 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.229167 6082 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.231498 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 88148a10290c4c48b87ab7e6aae5694c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:17.231881 2852 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.234795 2579 catalog_manager.cc:5484] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:17.239971 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet 88148a10290c4c48b87ab7e6aae5694c with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:17.240381 6060 tablet_replica.cc:330] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:17.240657 6060 raft_consensus.cc:2229] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:17.241115 6060 raft_consensus.cc:2258] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:17.243252 6060 ts_tablet_manager.cc:1875] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:17.243968 5029 raft_consensus.cc:2932] T 88148a10290c4c48b87ab7e6aae5694c P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.253985 6060 ts_tablet_manager.cc:1888] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:17.254151 6060 log.cc:1196] T 88148a10290c4c48b87ab7e6aae5694c P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/88148a10290c4c48b87ab7e6aae5694c
I20240502 14:16:17.254899 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet 88148a10290c4c48b87ab7e6aae5694c (table rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]) successfully deleted
I20240502 14:16:17.258675 6093 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:17.260241 6128 raft_consensus.cc:1050] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:17.260903 6128 consensus_queue.cc:237] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:17.259574 3111 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:17.265424 2576 catalog_manager.cc:5484] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 14:16:17.267838 2917 consensus_peers.cc:485] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 24027cd85c0d4480bd1cd4639f4f6540. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:17.272145 4765 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.273268 2722 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.273665 6135 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.274650 5552 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.274520 2851 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:17.286021 3111 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.287357 6128 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.288847 6128 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.295354 2722 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.296098 4765 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.299666 2579 catalog_manager.cc:5484] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:17.305780 2851 consensus_queue.cc:237] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:17.312548 2722 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.313122 6128 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.315749 5552 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.320468 2568 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e9b6901a9a524fd2ab1adb9d730fcd18 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:17.322561 2721 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.323204 2579 catalog_manager.cc:5484] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:17.329013 4766 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.329779 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet e9b6901a9a524fd2ab1adb9d730fcd18 with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:17.330207 6030 tablet_replica.cc:330] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:17.330428 6030 raft_consensus.cc:2229] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:17.330653 6030 pending_rounds.cc:62] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Trying to abort 1 pending ops.
I20240502 14:16:17.330771 6030 pending_rounds.cc:68] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7023244809258721280 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "e9b6901a9a524fd2ab1adb9d730fcd18" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } } }
I20240502 14:16:17.331048 6128 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.331252 6030 raft_consensus.cc:2894] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240502 14:16:17.331363 6030 raft_consensus.cc:2866] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 14:16:17.331460 6030 raft_consensus.cc:2258] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:17.334161 4765 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.336994 6030 ts_tablet_manager.cc:1875] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:17.345829 5324 raft_consensus.cc:1050] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:17.346911 5324 consensus_queue.cc:237] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:17.349359 6030 ts_tablet_manager.cc:1888] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:17.349591 6030 log.cc:1196] T e9b6901a9a524fd2ab1adb9d730fcd18 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/e9b6901a9a524fd2ab1adb9d730fcd18
I20240502 14:16:17.351169 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet e9b6901a9a524fd2ab1adb9d730fcd18 (table rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]) successfully deleted
I20240502 14:16:17.355465 2982 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.356356 6048 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.356549 2722 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.357451 4765 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.358583 5319 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.359139 5319 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.378557 6048 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.381203 2983 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.385298 2579 catalog_manager.cc:5484] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:17.388427 6187 raft_consensus.cc:979] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: : Instructing follower 6648fb2ed9f340ec9bd2d89ad88f2de4 to start an election
I20240502 14:16:17.388605 6187 raft_consensus.cc:1067] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Signalling peer 6648fb2ed9f340ec9bd2d89ad88f2de4 to start an election
I20240502 14:16:17.390127 4634 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb"
dest_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4"
from {username='slave'} at 127.27.220.132:48025
I20240502 14:16:17.390434 4634 raft_consensus.cc:482] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 14:16:17.390628 4634 raft_consensus.cc:3037] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:17.394330 2722 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.394481 4765 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.393316 4634 raft_consensus.cc:504] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:17.401609 4634 leader_election.cc:288] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [CANDIDATE]: Term 2 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:17.401772 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb" candidate_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:17.402030 2722 raft_consensus.cc:3037] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:17.402134 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb" candidate_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:17.402192 3111 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f3030ae5dc2425586db21b6667e2deb" candidate_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:17.402372 2982 raft_consensus.cc:3037] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:17.402431 3111 raft_consensus.cc:3032] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:16:17.402509 3111 raft_consensus.cc:729] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 472a7cc2e2d04cf0b36f7e43ed55f1d2, State: Running, Role: LEADER
I20240502 14:16:17.402817 3111 consensus_queue.cc:260] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:17.403270 3111 raft_consensus.cc:3037] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:17.405014 2722 raft_consensus.cc:2454] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6648fb2ed9f340ec9bd2d89ad88f2de4 in term 2.
I20240502 14:16:17.406340 3110 consensus_queue.cc:237] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:17.407946 2982 raft_consensus.cc:2454] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6648fb2ed9f340ec9bd2d89ad88f2de4 in term 2.
I20240502 14:16:17.407435 3111 raft_consensus.cc:2454] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6648fb2ed9f340ec9bd2d89ad88f2de4 in term 2.
I20240502 14:16:17.408490 2722 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.408429 4569 leader_election.cc:302] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6648fb2ed9f340ec9bd2d89ad88f2de4, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:17.408854 5745 raft_consensus.cc:2781] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:17.409150 6048 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.409471 4765 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.410730 6188 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.411834 6048 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.412659 2722 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.414007 4765 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.418704 2579 catalog_manager.cc:5484] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:17.419863 5745 raft_consensus.cc:686] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 LEADER]: Becoming Leader. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Running, Role: LEADER
I20240502 14:16:17.420706 5745 consensus_queue.cc:237] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:17.424044 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 609b87df71e64919a29a4fa093dfca23 with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:17.425066 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 609b87df71e64919a29a4fa093dfca23 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:17.427258 2576 catalog_manager.cc:5484] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: term changed from 1 to 2, leader changed from 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) to 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133). New cstate: current_term: 2 leader_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:17.427644 6217 tablet_replica.cc:330] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:17.427872 6217 raft_consensus.cc:2229] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:17.428241 6217 raft_consensus.cc:2258] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:17.429390 6217 ts_tablet_manager.cc:1875] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:17.435760 6217 ts_tablet_manager.cc:1888] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:17.436002 6217 log.cc:1196] T 609b87df71e64919a29a4fa093dfca23 P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/609b87df71e64919a29a4fa093dfca23
I20240502 14:16:17.436841 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 609b87df71e64919a29a4fa093dfca23 (table rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]) successfully deleted
I20240502 14:16:17.439529 2722 consensus_queue.cc:237] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:17.442005 3111 raft_consensus.cc:1261] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.442065 2982 raft_consensus.cc:1261] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.442592 6004 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 14:16:17.443327 2655 consensus_peers.cc:485] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 03ebc1eccb85405a8f5ea7582d8822f9. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:17.443670 6004 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.445858 5294 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:17.446614 3111 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:17.447335 2982 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:17.451426 2576 catalog_manager.cc:5484] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:17.464572 2982 consensus_queue.cc:237] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:17.467305 2851 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.467487 2722 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.468020 6082 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.468379 6083 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.471009 6082 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:17.471662 2722 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:17.478094 2576 catalog_manager.cc:5484] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:17.486534 2852 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
W20240502 14:16:17.492799 2917 consensus_peers.cc:485] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a975b36b40b74545a122ff11a6e4bee1. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:17.502292 2983 consensus_queue.cc:237] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:17.504544 2722 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.504704 2852 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.505095 6093 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.505354 6083 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 14:16:17.505776 2917 consensus_peers.cc:485] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 145a6a19b8474746a30ad51563419f1c. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:17.508360 6093 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:17.508996 2722 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:17.509503 2852 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:17.513995 2576 catalog_manager.cc:5484] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:17.527130 6188 raft_consensus.cc:1050] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:17.528052 6188 consensus_queue.cc:237] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:17.530473 4896 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.530526 2983 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.530522 2852 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:17.530998 6048 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.531320 5324 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.531649 5319 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:17.532138 2983 consensus_queue.cc:237] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:17.535734 3111 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 14:16:17.535984 2917 consensus_peers.cc:485] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3cc7001d1b9b425a8758f8e15914c289. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:17.536844 2851 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.537057 6083 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.537441 6082 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.551714 6082 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:17.555550 2853 consensus_queue.cc:237] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:17.555447 2850 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:17.558470 2722 raft_consensus.cc:1261] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.559240 6100 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.558195 5029 raft_consensus.cc:1261] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.562140 5552 consensus_queue.cc:1035] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.563563 5552 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.566826 2722 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.570256 2575 catalog_manager.cc:5484] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:17.574429 5029 raft_consensus.cc:2932] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.578465 2568 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3301dc9fca444af68c8a55dded735b38 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:17.582062 2576 catalog_manager.cc:5484] T 3301dc9fca444af68c8a55dded735b38 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:17.587069 6048 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.588984 3111 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:17.591141 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 3301dc9fca444af68c8a55dded735b38 with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:17.591630 6217 tablet_replica.cc:330] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:17.591862 6217 raft_consensus.cc:2229] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:17.592257 6217 raft_consensus.cc:2258] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:17.592468 4896 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.593868 6217 ts_tablet_manager.cc:1875] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:17.597900 2576 catalog_manager.cc:5484] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.600721 2852 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.602113 2982 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.607991 6217 ts_tablet_manager.cc:1888] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:17.608248 6217 log.cc:1196] T 3301dc9fca444af68c8a55dded735b38 P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/3301dc9fca444af68c8a55dded735b38
I20240502 14:16:17.611667 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 3301dc9fca444af68c8a55dded735b38 (table rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]) successfully deleted
I20240502 14:16:17.615715 3111 consensus_queue.cc:237] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:17.617558 4896 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.617746 2852 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:17.618271 6252 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.618716 6048 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.621927 5324 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.623230 2851 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.627173 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6cdd66f946db45478a93459ba69702b2 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:17.629768 2575 catalog_manager.cc:5484] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:17.631739 4896 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:17.634619 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 6cdd66f946db45478a93459ba69702b2 with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:17.634994 6217 tablet_replica.cc:330] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:17.635200 6217 raft_consensus.cc:2229] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:17.635439 6217 raft_consensus.cc:2258] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:17.636677 6217 ts_tablet_manager.cc:1875] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:17.644313 6217 ts_tablet_manager.cc:1888] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:17.644464 6217 log.cc:1196] T 6cdd66f946db45478a93459ba69702b2 P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/6cdd66f946db45478a93459ba69702b2
I20240502 14:16:17.645385 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 6cdd66f946db45478a93459ba69702b2 (table rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]) successfully deleted
I20240502 14:16:17.687549 6273 ts_tablet_manager.cc:902] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:17.691612 6273 tablet_copy_client.cc:308] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:17.692613 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 28f1bc6021e04a05916d3e49a8c98597 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:57321)
I20240502 14:16:17.693177 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 28f1bc6021e04a05916d3e49a8c98597 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:57321: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-28f1bc6021e04a05916d3e49a8c98597
I20240502 14:16:17.695845 3133 tablet_copy_source_session.cc:205] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:17.697175 6273 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28f1bc6021e04a05916d3e49a8c98597. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:17.702356 6273 tablet_copy_client.cc:783] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:17.702590 6273 tablet_copy_client.cc:647] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:17.704088 6283 tablet_copy_client.cc:1002] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:17.776682 6316 ts_tablet_manager.cc:902] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:17.782516 6316 tablet_copy_client.cc:308] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:17.790127 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet 24027cd85c0d4480bd1cd4639f4f6540 from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:33461)
I20240502 14:16:17.790323 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet 24027cd85c0d4480bd1cd4639f4f6540 from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:33461: session id = fa04f9da159541bfbbb6ac55db06e07b-24027cd85c0d4480bd1cd4639f4f6540
I20240502 14:16:17.793071 3003 tablet_copy_source_session.cc:205] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:17.794411 6316 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 24027cd85c0d4480bd1cd4639f4f6540. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:17.799786 6316 tablet_copy_client.cc:783] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:17.800102 6316 tablet_copy_client.cc:647] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:17.801795 6322 tablet_copy_client.cc:1002] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:17.804307 2722 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228"
dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:48808
I20240502 14:16:17.804562 2722 raft_consensus.cc:595] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:17.827991 3111 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:17.828665 6216 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.853382 6361 ts_tablet_manager.cc:902] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:17.855206 6361 tablet_copy_client.cc:308] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:17.856076 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 03ebc1eccb85405a8f5ea7582d8822f9 from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:35351)
I20240502 14:16:17.856247 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 03ebc1eccb85405a8f5ea7582d8822f9 from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:35351: session id = 763dcf9158e648b496b4faa1a0dc54af-03ebc1eccb85405a8f5ea7582d8822f9
I20240502 14:16:17.858876 2742 tablet_copy_source_session.cc:205] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:17.860491 6361 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 03ebc1eccb85405a8f5ea7582d8822f9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:17.868984 6361 tablet_copy_client.cc:783] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:17.869318 6361 tablet_copy_client.cc:647] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:17.871145 6365 tablet_copy_client.cc:1002] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:17.878291 2851 consensus_queue.cc:237] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:17.880753 2982 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.881485 3111 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.881386 6135 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.882072 6128 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.884512 6135 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:17.885177 2982 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:17.890430 3111 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:17.890836 2576 catalog_manager.cc:5484] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 14:16:17.893813 2787 consensus_peers.cc:485] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 56aa6c21e5e947baaeaa6b42c89dfb8f. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:17.907645 2982 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:17.908749 6380 ts_tablet_manager.cc:902] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:17.909549 6216 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.917623 6380 tablet_copy_client.cc:308] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:17.921170 2983 consensus_queue.cc:237] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
W20240502 14:16:17.923972 2916 consensus_peers.cc:485] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 33e74e4a8ff147d69a8051f0a898fae6. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:17.924019 2852 raft_consensus.cc:1261] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.924732 6083 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.925498 3111 raft_consensus.cc:1261] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:17.926461 6082 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:17.928766 5573 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:17.929625 6385 ts_tablet_manager.cc:902] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:17.930142 2852 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:17.936957 2579 catalog_manager.cc:5484] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:17.942327 3111 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:17.951704 6393 ts_tablet_manager.cc:902] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:17.951769 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet a975b36b40b74545a122ff11a6e4bee1 from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:37359)
I20240502 14:16:17.951942 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet a975b36b40b74545a122ff11a6e4bee1 from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:37359: session id = 21e5a60e02fd475bb47e5b3a2f294333-a975b36b40b74545a122ff11a6e4bee1
I20240502 14:16:17.954828 3003 tablet_copy_source_session.cc:205] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:17.956806 6380 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a975b36b40b74545a122ff11a6e4bee1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:17.963728 2722 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:17.964429 6216 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:17.971392 6380 tablet_copy_client.cc:783] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:17.971771 6380 tablet_copy_client.cc:647] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:17.978313 6384 tablet_copy_client.cc:1002] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:17.979996 6385 tablet_copy_client.cc:308] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:17.980962 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet 3cc7001d1b9b425a8758f8e15914c289 from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:33461)
I20240502 14:16:17.981195 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet 3cc7001d1b9b425a8758f8e15914c289 from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:33461: session id = fa04f9da159541bfbbb6ac55db06e07b-3cc7001d1b9b425a8758f8e15914c289
I20240502 14:16:17.985432 3003 tablet_copy_source_session.cc:205] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:17.988227 6385 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3cc7001d1b9b425a8758f8e15914c289. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:17.988693 6393 tablet_copy_client.cc:308] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:17.989384 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet 145a6a19b8474746a30ad51563419f1c from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:49647)
I20240502 14:16:17.989558 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet 145a6a19b8474746a30ad51563419f1c from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:49647: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-145a6a19b8474746a30ad51563419f1c
I20240502 14:16:17.991604 3003 tablet_copy_source_session.cc:205] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:17.992852 6393 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 145a6a19b8474746a30ad51563419f1c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:17.997129 6393 tablet_copy_client.cc:783] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:17.997186 6385 tablet_copy_client.cc:783] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:17.997406 6393 tablet_copy_client.cc:647] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:17.997452 6385 tablet_copy_client.cc:647] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:17.999485 6406 tablet_copy_client.cc:1002] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:18.000833 6397 tablet_copy_client.cc:1002] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:18.008299 2852 consensus_queue.cc:237] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:18.010661 2983 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:18.011009 2722 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:18.011255 6128 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:18.011704 5552 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:18.014914 5552 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:18.015540 2983 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
W20240502 14:16:18.017726 2787 consensus_peers.cc:485] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e4f8998b30c34530a308f78e61138315. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:18.021260 2579 catalog_manager.cc:5484] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:18.037041 2722 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:18.047181 2722 consensus_queue.cc:237] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:18.049826 2852 raft_consensus.cc:1261] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 14:16:18.050174 2655 consensus_peers.cc:485] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 169109a7069249df905c760be1866c67. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:18.050223 3111 raft_consensus.cc:1261] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:18.051162 5294 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.001s
I20240502 14:16:18.051601 6004 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:18.055301 6148 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:18.055945 2852 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:18.056902 3111 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:18.062150 2575 catalog_manager.cc:5484] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index -1 to 4, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:18.085928 5294 raft_consensus.cc:979] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: : Instructing follower 21e5a60e02fd475bb47e5b3a2f294333 to start an election
I20240502 14:16:18.086176 6004 raft_consensus.cc:1067] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Signalling peer 21e5a60e02fd475bb47e5b3a2f294333 to start an election
I20240502 14:16:18.087328 4896 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228"
dest_uuid: "21e5a60e02fd475bb47e5b3a2f294333"
from {username='slave'} at 127.27.220.129:40419
I20240502 14:16:18.087671 4896 raft_consensus.cc:482] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 14:16:18.087857 4896 raft_consensus.cc:3037] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:18.090936 4896 raft_consensus.cc:504] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:18.093482 3111 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:18.093791 3111 raft_consensus.cc:3037] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:18.094225 2722 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:18.094497 2722 raft_consensus.cc:3032] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:16:18.094638 2722 raft_consensus.cc:729] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:18.095059 2722 consensus_queue.cc:260] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:18.095594 2722 raft_consensus.cc:3037] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:18.096498 3111 raft_consensus.cc:2454] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:18.097249 4896 leader_election.cc:288] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: Requested vote from peers 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:18.099373 2722 raft_consensus.cc:2454] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:18.100057 4832 leader_election.cc:302] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 21e5a60e02fd475bb47e5b3a2f294333, 472a7cc2e2d04cf0b36f7e43ed55f1d2, 6b3499bcb84b45a094f7166639704ed4; no voters:
I20240502 14:16:18.100544 6251 raft_consensus.cc:2781] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:18.114328 6251 raft_consensus.cc:686] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 LEADER]: Becoming Leader. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Running, Role: LEADER
I20240502 14:16:18.115072 6251 consensus_queue.cc:237] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:18.119267 2852 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e46c5705a2aa4a3fbb410a284e5ea228" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:18.119542 2852 raft_consensus.cc:3037] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:18.122597 2852 raft_consensus.cc:2454] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:18.136147 2576 catalog_manager.cc:5484] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: term changed from 1 to 2, leader changed from 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) to 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135). New cstate: current_term: 2 leader_uuid: "21e5a60e02fd475bb47e5b3a2f294333" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:18.400956 4634 consensus_queue.cc:237] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:18.402902 2722 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 14:16:18.402936 2982 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 14:16:18.403545 6401 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:18.403856 6216 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:18.406206 5745 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:18.406914 2982 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:18.409427 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4f3030ae5dc2425586db21b6667e2deb with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:18.409683 2722 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:18.410965 2576 catalog_manager.cc:5484] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: config changed from index 5 to 7, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 2 leader_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:18.414791 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 4f3030ae5dc2425586db21b6667e2deb with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:18.421929 6486 tablet_replica.cc:330] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:18.422164 6486 raft_consensus.cc:2229] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:18.422384 6486 raft_consensus.cc:2258] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:18.423161 6486 ts_tablet_manager.cc:1875] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:18.427575 6486 ts_tablet_manager.cc:1888] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 14:16:18.427680 6486 log.cc:1196] T 4f3030ae5dc2425586db21b6667e2deb P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/4f3030ae5dc2425586db21b6667e2deb
I20240502 14:16:18.428467 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 4f3030ae5dc2425586db21b6667e2deb (table rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]) successfully deleted
I20240502 14:16:18.436007 6492 ts_tablet_manager.cc:902] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:18.438081 6492 tablet_copy_client.cc:308] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:44843
I20240502 14:16:18.438856 2873 tablet_copy_service.cc:140] P c6abc8535fe24d03bf71fb7092072fc4: Received BeginTabletCopySession request for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:40235)
I20240502 14:16:18.439045 2873 tablet_copy_service.cc:161] P c6abc8535fe24d03bf71fb7092072fc4: Beginning new tablet copy session on tablet 56aa6c21e5e947baaeaa6b42c89dfb8f from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:40235: session id = 21e5a60e02fd475bb47e5b3a2f294333-56aa6c21e5e947baaeaa6b42c89dfb8f
I20240502 14:16:18.441128 2873 tablet_copy_source_session.cc:205] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:18.442378 6492 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:18.447789 6492 tablet_copy_client.cc:783] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:18.448027 6492 tablet_copy_client.cc:647] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:18.449812 6496 tablet_copy_client.cc:1002] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:18.507109 6527 ts_tablet_manager.cc:902] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:18.508493 6527 tablet_copy_client.cc:308] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:18.509166 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet 33e74e4a8ff147d69a8051f0a898fae6 from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:55427)
I20240502 14:16:18.509311 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet 33e74e4a8ff147d69a8051f0a898fae6 from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:55427: session id = 763dcf9158e648b496b4faa1a0dc54af-33e74e4a8ff147d69a8051f0a898fae6
I20240502 14:16:18.512121 3003 tablet_copy_source_session.cc:205] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:18.513584 2722 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:18.513865 6527 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 33e74e4a8ff147d69a8051f0a898fae6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:18.514427 6251 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:18.520787 6527 tablet_copy_client.cc:783] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:18.521178 6527 tablet_copy_client.cc:647] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:18.523772 6531 tablet_copy_client.cc:1002] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:18.530186 6538 ts_tablet_manager.cc:902] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:18.532387 6538 tablet_copy_client.cc:308] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:44843
I20240502 14:16:18.539059 2873 tablet_copy_service.cc:140] P c6abc8535fe24d03bf71fb7092072fc4: Received BeginTabletCopySession request for tablet e4f8998b30c34530a308f78e61138315 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:35389)
I20240502 14:16:18.539263 2873 tablet_copy_service.cc:161] P c6abc8535fe24d03bf71fb7092072fc4: Beginning new tablet copy session on tablet e4f8998b30c34530a308f78e61138315 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:35389: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-e4f8998b30c34530a308f78e61138315
I20240502 14:16:18.541450 2873 tablet_copy_source_session.cc:205] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:18.542284 3111 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:18.542850 6538 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e4f8998b30c34530a308f78e61138315. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:18.543108 6251 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:18.549180 6538 tablet_copy_client.cc:783] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:18.549484 6538 tablet_copy_client.cc:647] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:18.551151 6547 tablet_copy_client.cc:1002] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:18.567420 2851 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:18.568266 6251 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:18.676240 6597 ts_tablet_manager.cc:902] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:18.677927 6597 tablet_copy_client.cc:308] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:18.678721 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 169109a7069249df905c760be1866c67 from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:35351)
I20240502 14:16:18.678918 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 169109a7069249df905c760be1866c67 from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:35351: session id = 763dcf9158e648b496b4faa1a0dc54af-169109a7069249df905c760be1866c67
I20240502 14:16:18.681767 2742 tablet_copy_source_session.cc:205] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:18.683621 6597 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 169109a7069249df905c760be1866c67. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:18.689692 6597 tablet_copy_client.cc:783] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:18.689950 6597 tablet_copy_client.cc:647] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:18.691560 6606 tablet_copy_client.cc:1002] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:19.142230 4896 consensus_queue.cc:237] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:19.143982 2852 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 14:16:19.144100 3111 raft_consensus.cc:1261] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 14:16:19.144686 6690 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:19.145048 6689 consensus_queue.cc:1035] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:19.147616 6689 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:19.148406 3111 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:19.150964 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e46c5705a2aa4a3fbb410a284e5ea228 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:19.151609 2852 raft_consensus.cc:2932] T e46c5705a2aa4a3fbb410a284e5ea228 P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:19.152612 2576 catalog_manager.cc:5484] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 5 to 7, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 2 leader_uuid: "21e5a60e02fd475bb47e5b3a2f294333" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:19.160286 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet e46c5705a2aa4a3fbb410a284e5ea228 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:19.160948 6695 tablet_replica.cc:330] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:19.161201 6695 raft_consensus.cc:2229] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:19.161466 6695 raft_consensus.cc:2258] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:19.162467 6695 ts_tablet_manager.cc:1875] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:19.166919 6695 ts_tablet_manager.cc:1888] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 14:16:19.167049 6695 log.cc:1196] T e46c5705a2aa4a3fbb410a284e5ea228 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/e46c5705a2aa4a3fbb410a284e5ea228
I20240502 14:16:19.168416 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet e46c5705a2aa4a3fbb410a284e5ea228 (table rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]) successfully deleted
I20240502 14:16:19.204551 6273 tablet_copy_client.cc:523] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:19.207742 6273 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:19.238332 6273 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:19.238790 6273 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:19.239070 6273 ts_tablet_manager.cc:1367] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.031s user 0.030s sys 0.003s
I20240502 14:16:19.239790 6273 raft_consensus.cc:348] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:19.240008 6273 raft_consensus.cc:729] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:19.240199 6273 consensus_queue.cc:260] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:19.240784 6273 ts_tablet_manager.cc:1398] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:19.241765 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-28f1bc6021e04a05916d3e49a8c98597 received from {username='slave'} at 127.27.220.133:57321
I20240502 14:16:19.241885 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-28f1bc6021e04a05916d3e49a8c98597 on tablet 28f1bc6021e04a05916d3e49a8c98597 with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:19.302217 6316 tablet_copy_client.cc:523] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:19.305927 6316 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:19.354182 6316 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:19.354799 6316 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:19.355090 6316 ts_tablet_manager.cc:1367] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.049s user 0.031s sys 0.012s
I20240502 14:16:19.355962 6316 raft_consensus.cc:348] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:19.356232 6316 raft_consensus.cc:729] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:19.356442 6316 consensus_queue.cc:260] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:19.357364 6316 ts_tablet_manager.cc:1398] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240502 14:16:19.358225 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-24027cd85c0d4480bd1cd4639f4f6540 received from {username='slave'} at 127.27.220.136:33461
I20240502 14:16:19.358376 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-24027cd85c0d4480bd1cd4639f4f6540 on tablet 24027cd85c0d4480bd1cd4639f4f6540 with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:19.371559 6361 tablet_copy_client.cc:523] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:19.374497 6361 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:19.419238 6361 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:19.419750 6361 tablet_bootstrap.cc:492] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:19.420073 6361 ts_tablet_manager.cc:1367] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.046s user 0.038s sys 0.004s
I20240502 14:16:19.422312 6361 raft_consensus.cc:348] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:19.422617 6361 raft_consensus.cc:729] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:19.422832 6361 consensus_queue.cc:260] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:19.423856 6361 ts_tablet_manager.cc:1398] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.004s user 0.005s sys 0.000s
I20240502 14:16:19.424737 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-03ebc1eccb85405a8f5ea7582d8822f9 received from {username='slave'} at 127.27.220.134:35351
I20240502 14:16:19.424932 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-03ebc1eccb85405a8f5ea7582d8822f9 on tablet 03ebc1eccb85405a8f5ea7582d8822f9 with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:19.478821 6380 tablet_copy_client.cc:523] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:19.481556 6380 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:19.500051 6393 tablet_copy_client.cc:523] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:19.501294 6385 tablet_copy_client.cc:523] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:19.503701 6393 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:19.506071 6385 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:19.537573 6380 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:19.538053 6380 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:19.538373 6380 ts_tablet_manager.cc:1367] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.057s user 0.037s sys 0.014s
I20240502 14:16:19.539278 6380 raft_consensus.cc:348] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:19.539559 6380 raft_consensus.cc:729] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:19.539795 6380 consensus_queue.cc:260] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:19.541093 6380 ts_tablet_manager.cc:1398] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.003s user 0.002s sys 0.001s
I20240502 14:16:19.541803 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-a975b36b40b74545a122ff11a6e4bee1 received from {username='slave'} at 127.27.220.135:37359
I20240502 14:16:19.541916 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-a975b36b40b74545a122ff11a6e4bee1 on tablet a975b36b40b74545a122ff11a6e4bee1 with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:19.563871 6393 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:19.564311 6393 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:19.564597 6393 ts_tablet_manager.cc:1367] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.061s user 0.046s sys 0.011s
I20240502 14:16:19.565399 6393 raft_consensus.cc:348] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:19.565683 6393 raft_consensus.cc:729] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:19.565918 6393 consensus_queue.cc:260] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:19.566473 6393 ts_tablet_manager.cc:1398] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:16:19.567301 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-145a6a19b8474746a30ad51563419f1c received from {username='slave'} at 127.27.220.133:49647
I20240502 14:16:19.567451 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-145a6a19b8474746a30ad51563419f1c on tablet 145a6a19b8474746a30ad51563419f1c with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:19.589373 4765 raft_consensus.cc:1203] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:19.595765 6385 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:19.596333 6385 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:19.597152 4896 raft_consensus.cc:1203] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:19.601570 6385 ts_tablet_manager.cc:1367] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.091s user 0.050s sys 0.008s
I20240502 14:16:19.602357 6385 raft_consensus.cc:348] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:19.602696 6385 raft_consensus.cc:729] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:19.602937 6385 consensus_queue.cc:260] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:19.603494 6385 ts_tablet_manager.cc:1398] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:19.604310 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-3cc7001d1b9b425a8758f8e15914c289 received from {username='slave'} at 127.27.220.136:33461
I20240502 14:16:19.604444 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-3cc7001d1b9b425a8758f8e15914c289 on tablet 3cc7001d1b9b425a8758f8e15914c289 with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:19.644135 5028 raft_consensus.cc:1203] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:19.662307 4634 raft_consensus.cc:1203] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:19.665608 5028 raft_consensus.cc:1203] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:19.719942 4634 raft_consensus.cc:1203] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:19.950174 6492 tablet_copy_client.cc:523] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:19.953101 6492 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:19.985745 6492 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:19.986289 6492 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:19.986580 6492 ts_tablet_manager.cc:1367] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.034s user 0.035s sys 0.000s
I20240502 14:16:19.987484 6492 raft_consensus.cc:348] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:19.987737 6492 raft_consensus.cc:729] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:19.987953 6492 consensus_queue.cc:260] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:19.988442 6492 ts_tablet_manager.cc:1398] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:19.989284 2873 tablet_copy_service.cc:342] P c6abc8535fe24d03bf71fb7092072fc4: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-56aa6c21e5e947baaeaa6b42c89dfb8f received from {username='slave'} at 127.27.220.135:40235
I20240502 14:16:19.989403 2873 tablet_copy_service.cc:433] P c6abc8535fe24d03bf71fb7092072fc4: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-56aa6c21e5e947baaeaa6b42c89dfb8f on tablet 56aa6c21e5e947baaeaa6b42c89dfb8f with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:20.006270 6148 raft_consensus.cc:1050] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:20.007184 6148 consensus_queue.cc:237] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:20.009569 4765 raft_consensus.cc:1261] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.009706 3111 raft_consensus.cc:1261] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.009738 2982 raft_consensus.cc:1261] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.010175 6862 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.010524 5294 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.010856 6004 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.014504 6862 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:20.015158 4765 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:20.016551 2983 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:20.020723 3111 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:20.020905 2577 catalog_manager.cc:5484] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:20.024227 6527 tablet_copy_client.cc:523] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:20.026229 6082 raft_consensus.cc:1050] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:20.027065 6082 consensus_queue.cc:237] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:20.029300 2722 consensus_queue.cc:237] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:20.031620 2982 raft_consensus.cc:1261] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.031683 2852 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.032184 4896 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.032457 4765 raft_consensus.cc:1261] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.032639 6083 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.032688 6004 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.032831 2722 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.033385 6083 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.033773 6004 consensus_queue.cc:1035] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.033934 6083 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.035339 6527 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:20.036134 6148 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:20.041390 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 03ebc1eccb85405a8f5ea7582d8822f9 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:20.042626 2579 catalog_manager.cc:5484] T 03ebc1eccb85405a8f5ea7582d8822f9 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:20.042733 4765 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:20.050009 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 03ebc1eccb85405a8f5ea7582d8822f9 with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:20.051584 6538 tablet_copy_client.cc:523] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:20.056301 6538 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:20.082887 2982 raft_consensus.cc:2932] T 03ebc1eccb85405a8f5ea7582d8822f9 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:20.099102 6870 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.100020 6875 tablet_replica.cc:330] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:20.099828 4897 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.100332 6875 raft_consensus.cc:2229] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:20.100652 6875 raft_consensus.cc:2258] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:20.105367 2575 catalog_manager.cc:5484] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.105979 2722 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.106796 6875 ts_tablet_manager.cc:1875] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:20.119271 2982 consensus_queue.cc:237] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:20.123052 4896 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.123642 6082 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.124797 2722 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.130887 6083 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.132275 6083 raft_consensus.cc:1050] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:20.133383 6083 consensus_queue.cc:237] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:20.136523 5573 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.142808 6877 raft_consensus.cc:1050] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:20.143692 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a975b36b40b74545a122ff11a6e4bee1 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:20.143611 6877 consensus_queue.cc:237] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:20.143954 6082 raft_consensus.cc:1050] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:20.143857 4896 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.145196 5028 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.145444 3111 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.145707 2850 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.146538 5573 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.148358 6082 consensus_queue.cc:237] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:20.148895 2722 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.149439 2575 catalog_manager.cc:5484] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.152695 5573 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.006s
I20240502 14:16:20.154752 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet a975b36b40b74545a122ff11a6e4bee1 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:20.161830 6870 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.168712 6884 raft_consensus.cc:1050] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:20.170586 2851 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.171306 6083 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.179857 2721 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.181593 4634 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.184370 2853 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.185325 6879 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.187284 6879 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20240502 14:16:20.188395 6876 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.191965 6597 tablet_copy_client.cc:523] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:20.196580 6597 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:20.203271 3111 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.203440 6538 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:20.203895 6538 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:20.204170 6538 ts_tablet_manager.cc:1367] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.148s user 0.054s sys 0.011s
I20240502 14:16:20.204036 6890 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.204999 6538 raft_consensus.cc:348] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:20.205241 6538 raft_consensus.cc:729] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:20.205457 6538 consensus_queue.cc:260] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:20.207830 6538 ts_tablet_manager.cc:1398] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:16:20.208446 2873 tablet_copy_service.cc:342] P c6abc8535fe24d03bf71fb7092072fc4: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-e4f8998b30c34530a308f78e61138315 received from {username='slave'} at 127.27.220.133:35389
I20240502 14:16:20.208541 2873 tablet_copy_service.cc:433] P c6abc8535fe24d03bf71fb7092072fc4: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-e4f8998b30c34530a308f78e61138315 on tablet e4f8998b30c34530a308f78e61138315 with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:20.210250 6082 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.216234 6875 ts_tablet_manager.cc:1888] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:20.216336 5028 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.218350 3110 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.218562 2575 catalog_manager.cc:5484] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:20.219684 6888 tablet_replica.cc:330] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:20.219889 6888 raft_consensus.cc:2229] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:20.220050 6888 pending_rounds.cc:62] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: Trying to abort 1 pending ops.
I20240502 14:16:20.220144 6888 pending_rounds.cc:68] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7023244820588642304 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "a975b36b40b74545a122ff11a6e4bee1" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } } }
I20240502 14:16:20.220587 6888 raft_consensus.cc:2894] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240502 14:16:20.220719 6888 raft_consensus.cc:2866] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 14:16:20.220825 6888 raft_consensus.cc:2258] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:20.224211 6875 log.cc:1196] T 03ebc1eccb85405a8f5ea7582d8822f9 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/03ebc1eccb85405a8f5ea7582d8822f9
I20240502 14:16:20.226703 6891 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.227607 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 03ebc1eccb85405a8f5ea7582d8822f9 (table rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]) successfully deleted
I20240502 14:16:20.231047 6888 ts_tablet_manager.cc:1875] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:20.237150 2852 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.239665 5029 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.248090 2982 consensus_queue.cc:237] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:20.244653 6884 consensus_queue.cc:237] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:20.251016 6888 ts_tablet_manager.cc:1888] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:20.251155 6888 log.cc:1196] T a975b36b40b74545a122ff11a6e4bee1 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/a975b36b40b74545a122ff11a6e4bee1
I20240502 14:16:20.251120 6870 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.254431 2853 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.257961 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet a975b36b40b74545a122ff11a6e4bee1 (table rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]) successfully deleted
I20240502 14:16:20.257475 6894 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.263237 5027 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.264752 2853 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.267084 2722 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.269924 6891 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.272212 2577 catalog_manager.cc:5484] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.273083 3111 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.275278 2575 catalog_manager.cc:5484] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:20.281994 6891 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.282315 4633 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.284866 6891 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.293852 3110 consensus_queue.cc:237] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:20.294412 2853 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.295107 6882 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.296252 4634 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.298301 2852 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.298702 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 24027cd85c0d4480bd1cd4639f4f6540 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:20.297561 4633 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.299811 6877 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.302014 6877 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.302512 2575 catalog_manager.cc:5484] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:20.304594 6876 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.305902 4896 raft_consensus.cc:1203] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:20.301805 3110 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.309726 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 28f1bc6021e04a05916d3e49a8c98597 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:20.313587 2577 catalog_manager.cc:5484] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:20.315001 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 24027cd85c0d4480bd1cd4639f4f6540 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:20.315369 6888 tablet_replica.cc:330] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:20.315608 6888 raft_consensus.cc:2229] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:20.315867 6888 raft_consensus.cc:2258] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:20.317569 6888 ts_tablet_manager.cc:1875] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:20.319005 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet 28f1bc6021e04a05916d3e49a8c98597 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:20.319258 2853 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.320196 4633 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.324687 6888 ts_tablet_manager.cc:1888] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:20.324898 6888 log.cc:1196] T 24027cd85c0d4480bd1cd4639f4f6540 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/24027cd85c0d4480bd1cd4639f4f6540
I20240502 14:16:20.325747 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 24027cd85c0d4480bd1cd4639f4f6540 (table rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]) successfully deleted
I20240502 14:16:20.334174 3111 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.337829 2722 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.337940 5028 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.340668 6870 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.340802 2982 consensus_queue.cc:237] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:20.341383 6870 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.344233 6597 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:20.344698 6597 tablet_bootstrap.cc:492] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:20.345000 6597 ts_tablet_manager.cc:1367] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.149s user 0.052s sys 0.017s
I20240502 14:16:20.345968 6597 raft_consensus.cc:348] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:20.346199 6597 raft_consensus.cc:729] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:20.346395 6597 consensus_queue.cc:260] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:20.347199 5028 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.347797 5029 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.351866 6597 ts_tablet_manager.cc:1398] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.007s user 0.004s sys 0.000s
I20240502 14:16:20.359045 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-169109a7069249df905c760be1866c67 received from {username='slave'} at 127.27.220.134:35351
I20240502 14:16:20.359268 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-169109a7069249df905c760be1866c67 on tablet 169109a7069249df905c760be1866c67 with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:20.360569 6882 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.361384 6907 tablet_replica.cc:330] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:20.361642 6907 raft_consensus.cc:2229] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:20.361264 6882 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.361974 6907 raft_consensus.cc:2258] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:20.368221 2853 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.369812 6907 ts_tablet_manager.cc:1875] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:20.369844 4634 raft_consensus.cc:1203] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:20.374087 3110 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.376956 6890 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.379101 2722 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.381649 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3cc7001d1b9b425a8758f8e15914c289 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:20.382284 5028 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.382915 6884 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.002s
I20240502 14:16:20.373064 2579 catalog_manager.cc:5484] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:20.387311 3110 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.390049 6907 ts_tablet_manager.cc:1888] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:20.392087 2578 catalog_manager.cc:5484] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b reported cstate change: config changed from index 5 to 6, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:20.398007 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 3cc7001d1b9b425a8758f8e15914c289 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:20.398406 6888 tablet_replica.cc:330] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:20.398659 6888 raft_consensus.cc:2229] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:20.399019 6888 raft_consensus.cc:2258] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:20.400041 6888 ts_tablet_manager.cc:1875] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:20.404716 6888 ts_tablet_manager.cc:1888] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:20.404847 6888 log.cc:1196] T 3cc7001d1b9b425a8758f8e15914c289 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/3cc7001d1b9b425a8758f8e15914c289
I20240502 14:16:20.405750 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 3cc7001d1b9b425a8758f8e15914c289 (table rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]) successfully deleted
I20240502 14:16:20.409729 6907 log.cc:1196] T 28f1bc6021e04a05916d3e49a8c98597 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/28f1bc6021e04a05916d3e49a8c98597
I20240502 14:16:20.410831 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet 28f1bc6021e04a05916d3e49a8c98597 (table rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]) successfully deleted
I20240502 14:16:20.419862 4633 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.425310 6527 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:20.425964 6527 tablet_bootstrap.cc:492] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:20.426352 6527 ts_tablet_manager.cc:1367] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.391s user 0.038s sys 0.011s
I20240502 14:16:20.427243 6527 raft_consensus.cc:348] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:20.427639 6527 raft_consensus.cc:729] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:20.427927 6527 consensus_queue.cc:260] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:20.429252 6527 ts_tablet_manager.cc:1398] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:16:20.430027 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-33e74e4a8ff147d69a8051f0a898fae6 received from {username='slave'} at 127.27.220.134:55427
I20240502 14:16:20.430202 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-33e74e4a8ff147d69a8051f0a898fae6 on tablet 33e74e4a8ff147d69a8051f0a898fae6 with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:20.478916 2982 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c"
dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:49890
I20240502 14:16:20.479179 2982 raft_consensus.cc:595] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:20.551743 4765 raft_consensus.cc:1203] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:20.569669 2982 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c"
dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:49930
I20240502 14:16:20.569926 2982 raft_consensus.cc:595] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:20.767758 4765 raft_consensus.cc:1203] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:20.823033 6916 raft_consensus.cc:1050] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:20.823941 6916 consensus_queue.cc:237] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:20.830041 2982 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c"
dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:49960
I20240502 14:16:20.830278 2982 raft_consensus.cc:595] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:20.834177 4633 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.834316 2982 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.834151 6633 raft_consensus.cc:1050] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:20.834915 7024 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.835310 7027 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.835529 6633 consensus_queue.cc:237] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:20.838126 2722 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.838563 6916 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.839046 2982 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.841627 7027 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.843322 3110 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.843717 2983 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.843837 4896 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:20.844040 4633 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.844931 2578 catalog_manager.cc:5484] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:20.846285 7027 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.846829 7027 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.848171 7026 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:20.855629 7031 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.856675 3110 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.859782 2983 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.865042 4896 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.866883 2579 catalog_manager.cc:5484] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.869027 2722 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:20.874881 2853 consensus_queue.cc:237] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:20.876694 4896 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.876880 3110 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:20.877488 7027 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.877815 6633 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:20.879222 6870 raft_consensus.cc:979] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: : Instructing follower 6b3499bcb84b45a094f7166639704ed4 to start an election
I20240502 14:16:20.879426 6870 raft_consensus.cc:1067] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Signalling peer 6b3499bcb84b45a094f7166639704ed4 to start an election
I20240502 14:16:20.880110 7027 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.880519 2722 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c"
dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
from {username='slave'} at 127.27.220.131:34889
I20240502 14:16:20.880785 2722 raft_consensus.cc:482] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 14:16:20.880919 2722 raft_consensus.cc:3037] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:20.880893 3110 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.882839 4896 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:20.883598 2568 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:20.884415 2722 raft_consensus.cc:504] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:20.885759 2853 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:20.885943 2853 raft_consensus.cc:3037] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:20.885886 2577 catalog_manager.cc:5484] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:20.888695 2853 raft_consensus.cc:2454] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 2.
I20240502 14:16:20.889847 2722 leader_election.cc:288] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 2 election: Requested vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
I20240502 14:16:20.890067 4633 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4"
I20240502 14:16:20.890293 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:20.890389 4633 raft_consensus.cc:3037] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:20.890981 7036 tablet_replica.cc:330] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:20.891166 7036 raft_consensus.cc:2229] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:20.891552 7036 raft_consensus.cc:2258] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:20.892333 7036 ts_tablet_manager.cc:1875] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:20.892616 4633 raft_consensus.cc:2454] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 2.
I20240502 14:16:20.893064 2656 leader_election.cc:302] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 6648fb2ed9f340ec9bd2d89ad88f2de4, 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:20.893414 6004 raft_consensus.cc:2781] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:20.893589 6004 raft_consensus.cc:686] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 2 LEADER]: Becoming Leader. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:20.893921 6004 consensus_queue.cc:237] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:20.895160 2983 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "145a6a19b8474746a30ad51563419f1c" candidate_uuid: "6b3499bcb84b45a094f7166639704ed4" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:20.895414 2983 raft_consensus.cc:3032] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:16:20.895504 2983 raft_consensus.cc:729] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:20.897382 2983 consensus_queue.cc:260] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:20.897451 2577 catalog_manager.cc:5484] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: term changed from 1 to 2, leader changed from 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) to 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129). New cstate: current_term: 2 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:20.897895 2983 raft_consensus.cc:3037] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:20.898303 7036 ts_tablet_manager.cc:1888] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:20.898432 7036 log.cc:1196] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/56aa6c21e5e947baaeaa6b42c89dfb8f
I20240502 14:16:20.899924 2983 raft_consensus.cc:2454] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6b3499bcb84b45a094f7166639704ed4 in term 2.
I20240502 14:16:20.900463 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 56aa6c21e5e947baaeaa6b42c89dfb8f (table rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]) successfully deleted
I20240502 14:16:21.105917 6890 raft_consensus.cc:1050] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:21.106786 6890 consensus_queue.cc:237] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:21.110569 3110 raft_consensus.cc:1261] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:21.111474 2853 raft_consensus.cc:1261] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:21.111383 6884 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:21.113090 4765 raft_consensus.cc:1261] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:21.113210 6884 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:21.113806 6890 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:21.124454 6884 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.126168 3111 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.134325 2853 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.135392 2577 catalog_manager.cc:5484] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.146268 4766 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.152709 2983 consensus_queue.cc:237] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:21.154824 4765 raft_consensus.cc:1261] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:21.154917 2853 raft_consensus.cc:1261] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:21.155896 6884 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:21.156167 6890 consensus_queue.cc:1035] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:21.158365 6870 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.159058 4765 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.160272 2853 raft_consensus.cc:2932] T 33e74e4a8ff147d69a8051f0a898fae6 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.161777 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 33e74e4a8ff147d69a8051f0a898fae6 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:21.163802 2575 catalog_manager.cc:5484] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:21.168462 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 33e74e4a8ff147d69a8051f0a898fae6 with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:21.169219 7077 tablet_replica.cc:330] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:21.169417 7077 raft_consensus.cc:2229] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:21.169696 7077 raft_consensus.cc:2258] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:21.170552 7077 ts_tablet_manager.cc:1875] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:21.175936 7077 ts_tablet_manager.cc:1888] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:21.176121 7077 log.cc:1196] T 33e74e4a8ff147d69a8051f0a898fae6 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/33e74e4a8ff147d69a8051f0a898fae6
I20240502 14:16:21.176874 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 33e74e4a8ff147d69a8051f0a898fae6 (table rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]) successfully deleted
I20240502 14:16:21.255721 2853 consensus_queue.cc:237] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:21.257730 4633 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:21.258198 2722 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:21.258306 7031 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:21.258862 7027 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:21.260664 7031 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:21.261173 4633 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:21.261772 2722 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:21.264945 2568 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e4f8998b30c34530a308f78e61138315 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:21.267513 2579 catalog_manager.cc:5484] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:21.271399 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet e4f8998b30c34530a308f78e61138315 with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:21.271746 7036 tablet_replica.cc:330] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:21.271977 7036 raft_consensus.cc:2229] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:21.272300 7036 raft_consensus.cc:2258] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:21.273075 7036 ts_tablet_manager.cc:1875] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:21.278040 7036 ts_tablet_manager.cc:1888] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:21.278167 7036 log.cc:1196] T e4f8998b30c34530a308f78e61138315 P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/e4f8998b30c34530a308f78e61138315
I20240502 14:16:21.278899 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet e4f8998b30c34530a308f78e61138315 (table rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]) successfully deleted
I20240502 14:16:21.354723 6148 raft_consensus.cc:1050] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:21.355629 6148 consensus_queue.cc:237] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:21.358217 2853 raft_consensus.cc:1261] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:21.358242 4765 raft_consensus.cc:1261] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:21.358467 3110 raft_consensus.cc:1261] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:21.358870 6148 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:21.359198 7105 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:21.359562 6004 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:21.363584 4765 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.363793 2853 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.367323 2853 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:21.362876 6148 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.368782 2578 catalog_manager.cc:5484] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.371686 6004 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:21.373947 3111 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.376957 2722 consensus_queue.cc:237] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:21.379356 4765 raft_consensus.cc:1261] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:21.380017 7105 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:21.381657 2851 raft_consensus.cc:1261] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:21.382228 7105 consensus_queue.cc:1035] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:21.382755 7105 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.383378 4765 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.386996 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 169109a7069249df905c760be1866c67 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:21.388196 2575 catalog_manager.cc:5484] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af reported cstate change: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.389684 2853 raft_consensus.cc:2932] T 169109a7069249df905c760be1866c67 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:21.391677 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 169109a7069249df905c760be1866c67 with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:21.392058 7077 tablet_replica.cc:330] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:21.392248 7077 raft_consensus.cc:2229] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:21.392557 7077 raft_consensus.cc:2258] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:21.393658 7077 ts_tablet_manager.cc:1875] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:21.397729 7077 ts_tablet_manager.cc:1888] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:21.397845 7077 log.cc:1196] T 169109a7069249df905c760be1866c67 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/169109a7069249df905c760be1866c67
I20240502 14:16:21.398519 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 169109a7069249df905c760be1866c67 (table rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]) successfully deleted
I20240502 14:16:21.422453 4633 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:21.423280 6004 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:21.431355 2983 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 14:16:21.432232 7105 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 14:16:21.435093 7105 consensus_queue.cc:1159] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "c6abc8535fe24d03bf71fb7092072fc4" responder_term: 2 status { last_received { term: 2 index: 6 } last_committed_idx: 6 last_received_current_leader { term: 2 index: 6 } }
I20240502 14:16:21.436071 2721 consensus_queue.cc:237] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:21.438346 4633 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 14:16:21.438396 2853 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 14:16:21.439049 7105 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.439604 6004 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.441654 7105 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:21.442194 4633 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:21.445480 2853 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:21.448187 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 145a6a19b8474746a30ad51563419f1c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:21.450403 2578 catalog_manager.cc:5484] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 5 to 7, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 2 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:21.454787 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 145a6a19b8474746a30ad51563419f1c with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:21.457583 7036 tablet_replica.cc:330] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:21.457837 7036 raft_consensus.cc:2229] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:21.458048 7036 raft_consensus.cc:2258] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:21.458892 7036 ts_tablet_manager.cc:1875] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:21.464983 7036 ts_tablet_manager.cc:1888] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 14:16:21.465081 7036 log.cc:1196] T 145a6a19b8474746a30ad51563419f1c P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/145a6a19b8474746a30ad51563419f1c
I20240502 14:16:21.465942 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 145a6a19b8474746a30ad51563419f1c (table rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]) successfully deleted
I20240502 14:16:21.610479 4878 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:21.612987 4614 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:21.615665 2701 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 14:16:21.632347 3091 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:21.645171 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:21.670065 5910 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:21.674268 2963 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:21.677277 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:21.748872 5009 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:21.845235 2721 consensus_queue.cc:237] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:21.847815 2983 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.848227 4633 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.848958 6148 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.849345 7105 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.853052 6148 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:21.857098 2983 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:21.859369 2572 catalog_manager.cc:5484] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 14:16:21.860970 2658 consensus_peers.cc:485] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 81be921c89194d3bb9c3d41be938ed74. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:21.871346 4633 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:21.886750 2721 consensus_queue.cc:237] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:21.889688 4896 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.889786 2853 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.890381 7103 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.890693 7105 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.894393 7103 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
W20240502 14:16:21.896173 2656 consensus_peers.cc:485] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cdaeb3ec3d344b4d9ac520f3d4c6d771. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:21.897310 4896 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:21.902290 2577 catalog_manager.cc:5484] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:21.904929 2853 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:21.923386 2983 consensus_queue.cc:237] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:21.926331 3111 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.927520 5028 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.928128 6890 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 14:16:21.928556 2917 consensus_peers.cc:485] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 24027cd85c0d4480bd1cd4639f4f6540. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:21.928881 6890 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.931905 6890 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:21.932579 5028 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:21.933845 3111 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:21.938764 2577 catalog_manager.cc:5484] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 6 to 7, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:21.955322 3111 consensus_queue.cc:237] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
W20240502 14:16:21.958079 3047 consensus_peers.cc:485] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9fd6780b78f849198bc91ed9ae2d08be. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:21.958127 2853 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.958387 5028 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.958644 7037 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.958963 7209 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.961577 7209 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:21.962213 2853 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:21.962386 5028 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:21.967553 2577 catalog_manager.cc:5484] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 6 to 7, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:21.986241 2983 consensus_queue.cc:237] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:21.988632 4896 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:21.988862 2721 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 14:16:21.989053 2916 consensus_peers.cc:485] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a975b36b40b74545a122ff11a6e4bee1. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:21.989248 7198 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.989611 6890 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:21.992420 7202 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:21.993039 2721 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:21.993170 4896 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:21.997411 2572 catalog_manager.cc:5484] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 6 to 7, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:22.012092 4633 consensus_queue.cc:237] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 2.7, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:22.015626 2721 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 14:16:22.016521 6401 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:22.029632 2983 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 14:16:22.030294 6401 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:22.036845 7227 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 LEADER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
W20240502 14:16:22.039450 4570 consensus_peers.cc:485] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4f3030ae5dc2425586db21b6667e2deb. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.043936 2721 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.046178 2983 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.051180 2578 catalog_manager.cc:5484] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 7 to 8, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 2 leader_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.064240 2721 consensus_queue.cc:237] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:22.066807 3111 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.067984 4633 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.067530 7105 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 14:16:22.069923 2656 consensus_peers.cc:485] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 652af8d083784c078c5b45cca32ece4b. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.070233 7105 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.071689 7103 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.072530 3111 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.078179 2577 catalog_manager.cc:5484] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:22.099617 4633 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.112411 2983 consensus_queue.cc:237] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:22.115006 3111 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.115589 4765 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.115628 7202 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 14:16:22.116106 2917 consensus_peers.cc:485] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e -> Peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Couldn't send request to peer 6648fb2ed9f340ec9bd2d89ad88f2de4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 02adec8c6c704326afbedb70578df917. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.117324 7202 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.119014 7202 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:22.122287 3111 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:22.125540 4765 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } } }
I20240502 14:16:22.126725 2572 catalog_manager.cc:5484] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 6 to 7, NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:22.160730 3111 consensus_queue.cc:237] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:22.163300 5028 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.163357 2851 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 14:16:22.163940 3046 consensus_peers.cc:485] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7f7589fa40dd4af09cc87e6e793bcd8a. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.163961 7210 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.164311 7037 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.167523 7210 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:22.168344 2851 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:22.168516 5028 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:22.173255 2572 catalog_manager.cc:5484] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b reported cstate change: config changed from index 6 to 7, NON_VOTER 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } } }
I20240502 14:16:22.188833 2721 consensus_queue.cc:237] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:22.191121 2851 raft_consensus.cc:1261] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 14:16:22.191371 2656 consensus_peers.cc:485] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6f785b6443b741a483e16eb154439b27. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.191395 2983 raft_consensus.cc:1261] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:16:22.191773 7103 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:22.192140 7105 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:16:22.194810 6148 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:22.195488 2983 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:22.198249 2851 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:22.200271 2576 catalog_manager.cc:5484] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index -1 to 4, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:22.215135 2851 consensus_queue.cc:237] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:22.217563 4633 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.218235 7026 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.221913 7026 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.225411 4633 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.232017 2576 catalog_manager.cc:5484] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.234915 2721 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.236698 7275 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 14:16:22.237464 2789 consensus_peers.cc:485] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e4f8998b30c34530a308f78e61138315. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.238500 2721 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.251433 3111 consensus_queue.cc:237] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:22.254202 4633 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.254245 2851 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 14:16:22.254415 3047 consensus_peers.cc:485] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 28f1bc6021e04a05916d3e49a8c98597. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.254788 7258 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.255142 7210 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.258102 7037 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.258565 2851 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.258904 4633 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.265077 2576 catalog_manager.cc:5484] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:22.283814 2851 consensus_queue.cc:237] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:22.286401 3111 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 14:16:22.286445 2789 consensus_peers.cc:485] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 56aa6c21e5e947baaeaa6b42c89dfb8f. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.287024 7275 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.287417 4896 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.287940 7275 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.290344 7275 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.291491 4896 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.291532 3111 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.297225 2577 catalog_manager.cc:5484] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.319473 7293 ts_tablet_manager.cc:902] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:22.323261 7293 tablet_copy_client.cc:308] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:22.328397 2721 consensus_queue.cc:237] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:22.333634 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:52803)
I20240502 14:16:22.333829 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:52803: session id = fa04f9da159541bfbbb6ac55db06e07b-cdaeb3ec3d344b4d9ac520f3d4c6d771
I20240502 14:16:22.337096 2742 tablet_copy_source_session.cc:205] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 14:16:22.338567 2656 consensus_peers.cc:485] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 -> Peer fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149): Couldn't send request to peer fa04f9da159541bfbbb6ac55db06e07b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: aa0b526e216f46f0a2fb95fe0be02c46. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.338801 7293 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.339165 2983 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.339197 4896 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.339793 7103 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.340358 7105 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.346355 7293 tablet_copy_client.cc:783] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.346673 7293 tablet_copy_client.cc:647] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.348783 7302 tablet_copy_client.cc:1002] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.356173 6148 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.356809 2983 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.362157 2572 catalog_manager.cc:5484] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.365502 4896 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } } }
I20240502 14:16:22.375986 7312 ts_tablet_manager.cc:902] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:22.378962 7312 tablet_copy_client.cc:308] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:22.379781 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet 24027cd85c0d4480bd1cd4639f4f6540 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:49647)
I20240502 14:16:22.379964 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet 24027cd85c0d4480bd1cd4639f4f6540 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:49647: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-24027cd85c0d4480bd1cd4639f4f6540
I20240502 14:16:22.382628 3003 tablet_copy_source_session.cc:205] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.382840 2721 consensus_queue.cc:237] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:22.384213 7312 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 24027cd85c0d4480bd1cd4639f4f6540. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.385354 2983 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.385399 4896 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:22.386328 7105 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
W20240502 14:16:22.386795 2658 consensus_peers.cc:485] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e882402ebdb24b03a9c51d74dcfd3550. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:22.387106 6148 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:22.389376 7105 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.390060 4896 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.391789 7312 tablet_copy_client.cc:783] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.392050 7312 tablet_copy_client.cc:647] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.392699 2983 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.396448 2576 catalog_manager.cc:5484] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:22.397210 7316 tablet_copy_client.cc:1002] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.470201 7360 ts_tablet_manager.cc:902] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:22.472844 7360 tablet_copy_client.cc:308] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:22.476136 7366 ts_tablet_manager.cc:902] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:22.486994 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 81be921c89194d3bb9c3d41be938ed74 from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:42751)
I20240502 14:16:22.487296 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 81be921c89194d3bb9c3d41be938ed74 from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:42751: session id = 6fb8f2ad13f24e81935b7df7faa291fe-81be921c89194d3bb9c3d41be938ed74
I20240502 14:16:22.491415 2742 tablet_copy_source_session.cc:205] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.493816 7369 ts_tablet_manager.cc:902] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:22.497354 7369 tablet_copy_client.cc:308] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:22.498463 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet a975b36b40b74545a122ff11a6e4bee1 from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:55427)
I20240502 14:16:22.498680 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet a975b36b40b74545a122ff11a6e4bee1 from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:55427: session id = 763dcf9158e648b496b4faa1a0dc54af-a975b36b40b74545a122ff11a6e4bee1
I20240502 14:16:22.498806 7360 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 81be921c89194d3bb9c3d41be938ed74. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.499092 7366 tablet_copy_client.cc:308] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:22.499800 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 9fd6780b78f849198bc91ed9ae2d08be from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:36283)
I20240502 14:16:22.499948 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 9fd6780b78f849198bc91ed9ae2d08be from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:36283: session id = 21e5a60e02fd475bb47e5b3a2f294333-9fd6780b78f849198bc91ed9ae2d08be
I20240502 14:16:22.501025 3003 tablet_copy_source_session.cc:205] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.502445 3133 tablet_copy_source_session.cc:205] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.502879 7369 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a975b36b40b74545a122ff11a6e4bee1. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.508585 7366 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9fd6780b78f849198bc91ed9ae2d08be. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.511516 7369 tablet_copy_client.cc:783] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.511780 7369 tablet_copy_client.cc:647] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.513487 7378 tablet_copy_client.cc:1002] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.514571 7360 tablet_copy_client.cc:783] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.514873 7360 tablet_copy_client.cc:647] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.515636 7366 tablet_copy_client.cc:783] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.516383 7366 tablet_copy_client.cc:647] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.516825 7364 tablet_copy_client.cc:1002] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.518328 7380 tablet_copy_client.cc:1002] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.590642 7411 ts_tablet_manager.cc:902] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:22.591776 7411 tablet_copy_client.cc:308] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:22.592442 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet 02adec8c6c704326afbedb70578df917 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 ({username='slave'} at 127.27.220.133:49647)
I20240502 14:16:22.592630 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet 02adec8c6c704326afbedb70578df917 from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 at {username='slave'} at 127.27.220.133:49647: session id = 6648fb2ed9f340ec9bd2d89ad88f2de4-02adec8c6c704326afbedb70578df917
I20240502 14:16:22.594719 3003 tablet_copy_source_session.cc:205] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.595813 7411 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 02adec8c6c704326afbedb70578df917. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.597689 7416 ts_tablet_manager.cc:902] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:22.601750 7411 tablet_copy_client.cc:783] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.602098 7411 tablet_copy_client.cc:647] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.603749 7415 tablet_copy_client.cc:1002] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.603919 7416 tablet_copy_client.cc:308] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:22.604588 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 652af8d083784c078c5b45cca32ece4b from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:52803)
I20240502 14:16:22.604804 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 652af8d083784c078c5b45cca32ece4b from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:52803: session id = fa04f9da159541bfbbb6ac55db06e07b-652af8d083784c078c5b45cca32ece4b
I20240502 14:16:22.607870 2742 tablet_copy_source_session.cc:205] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.609038 7416 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 652af8d083784c078c5b45cca32ece4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.615166 7416 tablet_copy_client.cc:783] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.615386 7416 tablet_copy_client.cc:647] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.616878 7420 tablet_copy_client.cc:1002] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.618638 7421 ts_tablet_manager.cc:902] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
I20240502 14:16:22.620198 7421 tablet_copy_client.cc:308] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:41841
I20240502 14:16:22.626520 4654 tablet_copy_service.cc:140] P 6648fb2ed9f340ec9bd2d89ad88f2de4: Received BeginTabletCopySession request for tablet 4f3030ae5dc2425586db21b6667e2deb from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:53377)
I20240502 14:16:22.626753 4654 tablet_copy_service.cc:161] P 6648fb2ed9f340ec9bd2d89ad88f2de4: Beginning new tablet copy session on tablet 4f3030ae5dc2425586db21b6667e2deb from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:53377: session id = 6fb8f2ad13f24e81935b7df7faa291fe-4f3030ae5dc2425586db21b6667e2deb
I20240502 14:16:22.627051 7426 ts_tablet_manager.cc:902] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:22.628700 7426 tablet_copy_client.cc:308] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:22.629380 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 6f785b6443b741a483e16eb154439b27 from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:51615)
I20240502 14:16:22.629554 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 6f785b6443b741a483e16eb154439b27 from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:51615: session id = 21e5a60e02fd475bb47e5b3a2f294333-6f785b6443b741a483e16eb154439b27
I20240502 14:16:22.629914 4654 tablet_copy_source_session.cc:205] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.631863 2742 tablet_copy_source_session.cc:205] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.631911 7421 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f3030ae5dc2425586db21b6667e2deb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.633100 7426 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6f785b6443b741a483e16eb154439b27. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.637934 7421 tablet_copy_client.cc:783] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.638156 7421 tablet_copy_client.cc:647] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.639015 7426 tablet_copy_client.cc:783] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.639245 7426 tablet_copy_client.cc:647] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.639989 7425 tablet_copy_client.cc:1002] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.640589 7430 tablet_copy_client.cc:1002] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.746872 7431 ts_tablet_manager.cc:902] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:22.749687 7431 tablet_copy_client.cc:308] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:44843
I20240502 14:16:22.756346 2873 tablet_copy_service.cc:140] P c6abc8535fe24d03bf71fb7092072fc4: Received BeginTabletCopySession request for tablet e4f8998b30c34530a308f78e61138315 from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:47391)
I20240502 14:16:22.756515 2873 tablet_copy_service.cc:161] P c6abc8535fe24d03bf71fb7092072fc4: Beginning new tablet copy session on tablet e4f8998b30c34530a308f78e61138315 from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:47391: session id = 6fb8f2ad13f24e81935b7df7faa291fe-e4f8998b30c34530a308f78e61138315
I20240502 14:16:22.758987 2873 tablet_copy_source_session.cc:205] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.760600 7431 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e4f8998b30c34530a308f78e61138315. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.763553 7436 ts_tablet_manager.cc:902] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:22.767136 7431 tablet_copy_client.cc:783] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.767392 7431 tablet_copy_client.cc:647] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.769568 7435 tablet_copy_client.cc:1002] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.772940 7436 tablet_copy_client.cc:308] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:22.773705 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet e882402ebdb24b03a9c51d74dcfd3550 from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:42751)
I20240502 14:16:22.773906 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet e882402ebdb24b03a9c51d74dcfd3550 from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:42751: session id = 6fb8f2ad13f24e81935b7df7faa291fe-e882402ebdb24b03a9c51d74dcfd3550
I20240502 14:16:22.776315 2742 tablet_copy_source_session.cc:205] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.777427 7446 ts_tablet_manager.cc:902] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:22.778060 7436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e882402ebdb24b03a9c51d74dcfd3550. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.785233 7436 tablet_copy_client.cc:783] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.785573 7436 tablet_copy_client.cc:647] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.787782 7440 tablet_copy_client.cc:1002] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.789860 7446 tablet_copy_client.cc:308] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:22.790557 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 7f7589fa40dd4af09cc87e6e793bcd8a from peer 763dcf9158e648b496b4faa1a0dc54af ({username='slave'} at 127.27.220.134:42215)
I20240502 14:16:22.790711 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 7f7589fa40dd4af09cc87e6e793bcd8a from peer 763dcf9158e648b496b4faa1a0dc54af at {username='slave'} at 127.27.220.134:42215: session id = 763dcf9158e648b496b4faa1a0dc54af-7f7589fa40dd4af09cc87e6e793bcd8a
I20240502 14:16:22.793388 3133 tablet_copy_source_session.cc:205] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.794557 7446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7f7589fa40dd4af09cc87e6e793bcd8a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.799799 7446 tablet_copy_client.cc:783] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.800040 7446 tablet_copy_client.cc:647] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.801586 7455 tablet_copy_client.cc:1002] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.821065 7472 ts_tablet_manager.cc:902] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:22.822832 7472 tablet_copy_client.cc:308] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:22.823649 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 28f1bc6021e04a05916d3e49a8c98597 from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:32937)
I20240502 14:16:22.823858 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 28f1bc6021e04a05916d3e49a8c98597 from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:32937: session id = fa04f9da159541bfbbb6ac55db06e07b-28f1bc6021e04a05916d3e49a8c98597
I20240502 14:16:22.826491 3133 tablet_copy_source_session.cc:205] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.828385 7472 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 28f1bc6021e04a05916d3e49a8c98597. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.833626 7472 tablet_copy_client.cc:783] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.833873 7472 tablet_copy_client.cc:647] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.835440 7476 tablet_copy_client.cc:1002] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.882205 7502 ts_tablet_manager.cc:902] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:22.884575 7502 tablet_copy_client.cc:308] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:44843
I20240502 14:16:22.885461 2873 tablet_copy_service.cc:140] P c6abc8535fe24d03bf71fb7092072fc4: Received BeginTabletCopySession request for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:47391)
I20240502 14:16:22.885661 2873 tablet_copy_service.cc:161] P c6abc8535fe24d03bf71fb7092072fc4: Beginning new tablet copy session on tablet 56aa6c21e5e947baaeaa6b42c89dfb8f from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:47391: session id = 6fb8f2ad13f24e81935b7df7faa291fe-56aa6c21e5e947baaeaa6b42c89dfb8f
I20240502 14:16:22.888078 2873 tablet_copy_source_session.cc:205] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.889853 7502 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.895157 7502 tablet_copy_client.cc:783] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.895406 7502 tablet_copy_client.cc:647] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.897230 7511 tablet_copy_client.cc:1002] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:22.948331 7532 ts_tablet_manager.cc:902] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:22.949718 7532 tablet_copy_client.cc:308] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:22.950377 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet aa0b526e216f46f0a2fb95fe0be02c46 from peer fa04f9da159541bfbbb6ac55db06e07b ({username='slave'} at 127.27.220.136:52803)
I20240502 14:16:22.950546 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet aa0b526e216f46f0a2fb95fe0be02c46 from peer fa04f9da159541bfbbb6ac55db06e07b at {username='slave'} at 127.27.220.136:52803: session id = fa04f9da159541bfbbb6ac55db06e07b-aa0b526e216f46f0a2fb95fe0be02c46
I20240502 14:16:22.952524 2742 tablet_copy_source_session.cc:205] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:22.953733 7532 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa0b526e216f46f0a2fb95fe0be02c46. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:22.958873 7532 tablet_copy_client.cc:783] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:22.959102 7532 tablet_copy_client.cc:647] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:22.960460 7536 tablet_copy_client.cc:1002] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:23.849198 7293 tablet_copy_client.cc:523] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:23.852699 7293 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:23.892078 7293 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:23.892581 7293 tablet_bootstrap.cc:492] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:23.892906 7293 ts_tablet_manager.cc:1367] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.040s user 0.039s sys 0.000s
I20240502 14:16:23.893759 7293 raft_consensus.cc:348] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:23.894059 7293 raft_consensus.cc:729] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:23.894295 7293 consensus_queue.cc:260] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:23.895175 7293 ts_tablet_manager.cc:1398] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.002s user 0.005s sys 0.000s
I20240502 14:16:23.895857 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-cdaeb3ec3d344b4d9ac520f3d4c6d771 received from {username='slave'} at 127.27.220.136:52803
I20240502 14:16:23.896025 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-cdaeb3ec3d344b4d9ac520f3d4c6d771 on tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:23.898170 7312 tablet_copy_client.cc:523] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:23.902539 7312 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:23.950680 7312 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:23.951177 7312 tablet_bootstrap.cc:492] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:23.951501 7312 ts_tablet_manager.cc:1367] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.049s user 0.045s sys 0.001s
I20240502 14:16:23.952188 7312 raft_consensus.cc:348] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:23.952405 7312 raft_consensus.cc:729] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:23.952541 7312 consensus_queue.cc:260] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:23.953083 7312 ts_tablet_manager.cc:1398] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 14:16:23.953984 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-24027cd85c0d4480bd1cd4639f4f6540 received from {username='slave'} at 127.27.220.133:49647
I20240502 14:16:23.954164 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-24027cd85c0d4480bd1cd4639f4f6540 on tablet 24027cd85c0d4480bd1cd4639f4f6540 with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:24.013963 7369 tablet_copy_client.cc:523] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.016780 7369 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:24.017365 7360 tablet_copy_client.cc:523] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.018777 7366 tablet_copy_client.cc:523] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.022157 7366 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:24.022259 7360 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:24.036403 7360 log.cc:824] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:24.057541 7366 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.058092 7366 tablet_bootstrap.cc:492] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:24.058377 7366 ts_tablet_manager.cc:1367] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.036s user 0.030s sys 0.005s
I20240502 14:16:24.059065 7366 raft_consensus.cc:348] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:24.059288 7366 raft_consensus.cc:729] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:24.059479 7366 consensus_queue.cc:260] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:24.060122 7366 ts_tablet_manager.cc:1398] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:16:24.060945 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-9fd6780b78f849198bc91ed9ae2d08be received from {username='slave'} at 127.27.220.135:36283
I20240502 14:16:24.061101 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-9fd6780b78f849198bc91ed9ae2d08be on tablet 9fd6780b78f849198bc91ed9ae2d08be with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:24.070281 7369 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.070711 7369 tablet_bootstrap.cc:492] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:24.071053 7369 ts_tablet_manager.cc:1367] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.054s user 0.048s sys 0.004s
I20240502 14:16:24.071877 7369 raft_consensus.cc:348] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:24.072300 7369 raft_consensus.cc:729] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:24.072508 7369 consensus_queue.cc:260] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:24.078127 7369 ts_tablet_manager.cc:1398] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.007s user 0.004s sys 0.004s
I20240502 14:16:24.079392 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-a975b36b40b74545a122ff11a6e4bee1 received from {username='slave'} at 127.27.220.134:55427
I20240502 14:16:24.079567 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-a975b36b40b74545a122ff11a6e4bee1 on tablet a975b36b40b74545a122ff11a6e4bee1 with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:24.089368 7360 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.089890 7360 tablet_bootstrap.cc:492] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:24.090296 7360 ts_tablet_manager.cc:1367] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.068s user 0.063s sys 0.000s
I20240502 14:16:24.094870 7360 raft_consensus.cc:348] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.095230 7360 raft_consensus.cc:729] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:24.095642 7360 consensus_queue.cc:260] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.097124 7360 ts_tablet_manager.cc:1398] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.007s user 0.007s sys 0.000s
I20240502 14:16:24.098035 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-81be921c89194d3bb9c3d41be938ed74 received from {username='slave'} at 127.27.220.137:42751
I20240502 14:16:24.098167 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-81be921c89194d3bb9c3d41be938ed74 on tablet 81be921c89194d3bb9c3d41be938ed74 with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:24.104143 7411 tablet_copy_client.cc:523] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.106876 7411 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap starting.
I20240502 14:16:24.114806 4896 raft_consensus.cc:1203] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.117233 7416 tablet_copy_client.cc:523] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.120221 7416 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:24.123823 5028 raft_consensus.cc:1203] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.131325 4765 raft_consensus.cc:1203] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.140362 7421 tablet_copy_client.cc:523] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.141027 7426 tablet_copy_client.cc:523] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.143849 7421 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:24.144377 7426 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:24.162175 7411 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.176723 7416 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.177160 7416 tablet_bootstrap.cc:492] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:24.177479 7416 ts_tablet_manager.cc:1367] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.057s user 0.041s sys 0.007s
I20240502 14:16:24.178010 7411 tablet_bootstrap.cc:492] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Bootstrap complete.
I20240502 14:16:24.178365 7411 ts_tablet_manager.cc:1367] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent bootstrapping tablet: real 0.072s user 0.036s sys 0.008s
I20240502 14:16:24.178270 7416 raft_consensus.cc:348] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:24.178566 7416 raft_consensus.cc:729] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:24.178804 7416 consensus_queue.cc:260] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:24.179529 7411 raft_consensus.cc:348] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:24.179877 7411 raft_consensus.cc:729] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6648fb2ed9f340ec9bd2d89ad88f2de4, State: Initialized, Role: LEARNER
I20240502 14:16:24.180187 7416 ts_tablet_manager.cc:1398] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 14:16:24.180135 7411 consensus_queue.cc:260] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: NON_VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: true } }
I20240502 14:16:24.180858 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-652af8d083784c078c5b45cca32ece4b received from {username='slave'} at 127.27.220.136:52803
I20240502 14:16:24.181008 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-652af8d083784c078c5b45cca32ece4b on tablet 652af8d083784c078c5b45cca32ece4b with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:24.182569 7411 ts_tablet_manager.cc:1398] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:16:24.183347 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-02adec8c6c704326afbedb70578df917 received from {username='slave'} at 127.27.220.133:49647
I20240502 14:16:24.183502 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 6648fb2ed9f340ec9bd2d89ad88f2de4-02adec8c6c704326afbedb70578df917 on tablet 02adec8c6c704326afbedb70578df917 with peer 6648fb2ed9f340ec9bd2d89ad88f2de4
I20240502 14:16:24.187568 7421 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.187959 7421 tablet_bootstrap.cc:492] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:24.188221 7421 ts_tablet_manager.cc:1367] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.044s user 0.035s sys 0.004s
I20240502 14:16:24.189116 7421 raft_consensus.cc:348] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.189342 7421 raft_consensus.cc:729] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:24.189546 7421 consensus_queue.cc:260] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.191283 7421 ts_tablet_manager.cc:1398] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.003s user 0.004s sys 0.000s
I20240502 14:16:24.191944 4654 tablet_copy_service.cc:342] P 6648fb2ed9f340ec9bd2d89ad88f2de4: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-4f3030ae5dc2425586db21b6667e2deb received from {username='slave'} at 127.27.220.137:53377
I20240502 14:16:24.192164 4654 tablet_copy_service.cc:433] P 6648fb2ed9f340ec9bd2d89ad88f2de4: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-4f3030ae5dc2425586db21b6667e2deb on tablet 4f3030ae5dc2425586db21b6667e2deb with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:24.198408 7426 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.198768 7426 tablet_bootstrap.cc:492] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:24.199025 7426 ts_tablet_manager.cc:1367] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.055s user 0.049s sys 0.000s
I20240502 14:16:24.199681 7426 raft_consensus.cc:348] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:24.199883 7426 raft_consensus.cc:729] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:24.200071 7426 consensus_queue.cc:260] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:24.200671 7426 ts_tablet_manager.cc:1398] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 14:16:24.201346 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-6f785b6443b741a483e16eb154439b27 received from {username='slave'} at 127.27.220.135:51615
I20240502 14:16:24.201494 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-6f785b6443b741a483e16eb154439b27 on tablet 6f785b6443b741a483e16eb154439b27 with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:24.222460 4633 raft_consensus.cc:1203] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.258960 4633 raft_consensus.cc:1203] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.270198 7431 tablet_copy_client.cc:523] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.273456 7431 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:24.288503 7436 tablet_copy_client.cc:523] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.291983 7436 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:24.302009 7446 tablet_copy_client.cc:523] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.307957 7446 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap starting.
I20240502 14:16:24.336131 7472 tablet_copy_client.cc:523] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.338371 5932 raft_consensus.cc:1203] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEARNER]: Deduplicated request from leader. Original: 2.7->[2.8-2.8] Dedup: 2.8->[]
I20240502 14:16:24.339813 7472 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:24.358204 7431 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.358793 7431 tablet_bootstrap.cc:492] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:24.359203 7431 ts_tablet_manager.cc:1367] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.086s user 0.054s sys 0.009s
I20240502 14:16:24.360343 7431 raft_consensus.cc:348] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.360716 7431 raft_consensus.cc:729] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:24.360965 7431 consensus_queue.cc:260] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.364519 7431 ts_tablet_manager.cc:1398] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 14:16:24.367360 2873 tablet_copy_service.cc:342] P c6abc8535fe24d03bf71fb7092072fc4: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-e4f8998b30c34530a308f78e61138315 received from {username='slave'} at 127.27.220.137:47391
I20240502 14:16:24.367504 2873 tablet_copy_service.cc:433] P c6abc8535fe24d03bf71fb7092072fc4: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-e4f8998b30c34530a308f78e61138315 on tablet e4f8998b30c34530a308f78e61138315 with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:24.376148 5932 raft_consensus.cc:1203] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.398089 7436 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.398231 7502 tablet_copy_client.cc:523] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.402767 7502 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:24.409584 7472 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.410061 7472 tablet_bootstrap.cc:492] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:24.410328 7472 ts_tablet_manager.cc:1367] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.071s user 0.048s sys 0.006s
I20240502 14:16:24.412185 7472 raft_consensus.cc:348] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:24.412508 7472 raft_consensus.cc:729] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:24.412719 7472 consensus_queue.cc:260] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:24.413468 7436 tablet_bootstrap.cc:492] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:24.415496 7446 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.416013 7446 tablet_bootstrap.cc:492] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: Bootstrap complete.
I20240502 14:16:24.415994 7436 ts_tablet_manager.cc:1367] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.124s user 0.070s sys 0.014s
I20240502 14:16:24.416168 7472 ts_tablet_manager.cc:1398] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:16:24.416332 7446 ts_tablet_manager.cc:1367] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: Time spent bootstrapping tablet: real 0.109s user 0.050s sys 0.011s
I20240502 14:16:24.416894 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-28f1bc6021e04a05916d3e49a8c98597 received from {username='slave'} at 127.27.220.136:32937
I20240502 14:16:24.417045 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-28f1bc6021e04a05916d3e49a8c98597 on tablet 28f1bc6021e04a05916d3e49a8c98597 with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:24.416947 7436 raft_consensus.cc:348] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.417096 7446 raft_consensus.cc:348] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:24.417292 7436 raft_consensus.cc:729] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:24.417343 7446 raft_consensus.cc:729] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 763dcf9158e648b496b4faa1a0dc54af, State: Initialized, Role: LEARNER
I20240502 14:16:24.417587 7446 consensus_queue.cc:260] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: true } }
I20240502 14:16:24.417560 7436 consensus_queue.cc:260] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.418313 7446 ts_tablet_manager.cc:1398] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:16:24.419476 7436 ts_tablet_manager.cc:1398] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 14:16:24.420262 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-e882402ebdb24b03a9c51d74dcfd3550 received from {username='slave'} at 127.27.220.137:42751
I20240502 14:16:24.420419 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-e882402ebdb24b03a9c51d74dcfd3550 on tablet e882402ebdb24b03a9c51d74dcfd3550 with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:24.421505 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 763dcf9158e648b496b4faa1a0dc54af-7f7589fa40dd4af09cc87e6e793bcd8a received from {username='slave'} at 127.27.220.134:42215
I20240502 14:16:24.421679 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 763dcf9158e648b496b4faa1a0dc54af-7f7589fa40dd4af09cc87e6e793bcd8a on tablet 7f7589fa40dd4af09cc87e6e793bcd8a with peer 763dcf9158e648b496b4faa1a0dc54af
I20240502 14:16:24.453954 5932 raft_consensus.cc:1203] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.459993 7502 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.460454 7502 tablet_bootstrap.cc:492] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:24.460708 7502 ts_tablet_manager.cc:1367] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.058s user 0.040s sys 0.004s
I20240502 14:16:24.460842 7532 tablet_copy_client.cc:523] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:24.461342 7502 raft_consensus.cc:348] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.461591 7502 raft_consensus.cc:729] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:24.461789 7502 consensus_queue.cc:260] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:24.462311 7502 ts_tablet_manager.cc:1398] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.001s user 0.000s sys 0.004s
I20240502 14:16:24.463145 2873 tablet_copy_service.cc:342] P c6abc8535fe24d03bf71fb7092072fc4: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-56aa6c21e5e947baaeaa6b42c89dfb8f received from {username='slave'} at 127.27.220.137:47391
I20240502 14:16:24.463315 2873 tablet_copy_service.cc:433] P c6abc8535fe24d03bf71fb7092072fc4: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-56aa6c21e5e947baaeaa6b42c89dfb8f on tablet 56aa6c21e5e947baaeaa6b42c89dfb8f with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:24.464041 7532 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap starting.
I20240502 14:16:24.475378 5932 raft_consensus.cc:1203] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.500778 7532 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:24.501214 7532 tablet_bootstrap.cc:492] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: Bootstrap complete.
I20240502 14:16:24.501448 7532 ts_tablet_manager.cc:1367] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: Time spent bootstrapping tablet: real 0.038s user 0.032s sys 0.003s
I20240502 14:16:24.502102 7532 raft_consensus.cc:348] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:24.502326 7532 raft_consensus.cc:729] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Initialized, Role: LEARNER
I20240502 14:16:24.502513 7532 consensus_queue.cc:260] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: true } }
I20240502 14:16:24.502939 7532 ts_tablet_manager.cc:1398] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 14:16:24.503696 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session fa04f9da159541bfbbb6ac55db06e07b-aa0b526e216f46f0a2fb95fe0be02c46 received from {username='slave'} at 127.27.220.136:52803
I20240502 14:16:24.503844 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session fa04f9da159541bfbbb6ac55db06e07b-aa0b526e216f46f0a2fb95fe0be02c46 on tablet aa0b526e216f46f0a2fb95fe0be02c46 with peer fa04f9da159541bfbbb6ac55db06e07b
I20240502 14:16:24.560663 5932 raft_consensus.cc:1203] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.565258 7864 raft_consensus.cc:1050] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:24.565927 7864 consensus_queue.cc:237] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:24.568403 4896 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.568431 2721 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.568598 4765 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.569028 7865 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.569342 7867 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.569700 7864 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.572743 7864 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:24.573576 4765 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:24.576017 4896 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:24.577338 2572 catalog_manager.cc:5484] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:24.581070 5028 raft_consensus.cc:1203] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.583403 2721 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:24.587394 2983 consensus_queue.cc:237] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:24.589322 4896 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:24.590126 4765 raft_consensus.cc:1261] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:24.590626 7865 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.590878 7867 consensus_queue.cc:1035] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.593261 7867 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:24.593837 4896 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:24.593855 4765 raft_consensus.cc:2932] T a975b36b40b74545a122ff11a6e4bee1 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:24.596909 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a975b36b40b74545a122ff11a6e4bee1 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:24.598953 2577 catalog_manager.cc:5484] T a975b36b40b74545a122ff11a6e4bee1 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:24.604151 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet a975b36b40b74545a122ff11a6e4bee1 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:24.605041 7875 tablet_replica.cc:330] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:24.605232 7875 raft_consensus.cc:2229] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:24.605482 7875 raft_consensus.cc:2258] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:24.606377 7875 ts_tablet_manager.cc:1875] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:24.610656 7875 ts_tablet_manager.cc:1888] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:24.610769 7875 log.cc:1196] T a975b36b40b74545a122ff11a6e4bee1 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/a975b36b40b74545a122ff11a6e4bee1
I20240502 14:16:24.611460 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet a975b36b40b74545a122ff11a6e4bee1 (table rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]) successfully deleted
I20240502 14:16:24.626466 6148 raft_consensus.cc:1050] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:24.627167 6148 consensus_queue.cc:237] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:24.629987 4897 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.630020 2851 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.630146 5028 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.630795 7878 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.631157 6148 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.631469 7876 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.634897 7878 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:24.635396 2851 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:24.636042 4897 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:24.639784 5028 raft_consensus.cc:1203] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.640873 2577 catalog_manager.cc:5484] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:24.644817 5028 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:24.660346 7202 raft_consensus.cc:1050] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:24.661351 7202 consensus_queue.cc:237] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:24.669394 4633 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.670097 7864 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.671260 3110 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.671824 7202 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.673667 5028 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.674454 7867 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.690743 4897 raft_consensus.cc:1203] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:16:24.704440 7891 raft_consensus.cc:1050] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:24.705030 7865 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.705921 7891 consensus_queue.cc:237] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:24.705847 3110 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.706545 4634 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.710891 5028 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.714438 2578 catalog_manager.cc:5484] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.715188 4897 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.716244 5029 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.717008 7894 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.717806 2851 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.721501 7891 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.004s
I20240502 14:16:24.722293 7894 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.723385 2983 consensus_queue.cc:237] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:24.726732 5029 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:24.727301 4633 raft_consensus.cc:1261] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:24.727988 7864 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.729653 7864 consensus_queue.cc:1035] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.733980 7864 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.737118 5029 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.742069 4633 raft_consensus.cc:2932] T 24027cd85c0d4480bd1cd4639f4f6540 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.748827 2576 catalog_manager.cc:5484] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b reported cstate change: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.752362 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 24027cd85c0d4480bd1cd4639f4f6540 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:24.756577 4766 raft_consensus.cc:1203] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.763666 5029 raft_consensus.cc:1203] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:24.768569 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 24027cd85c0d4480bd1cd4639f4f6540 with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:24.769006 5029 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.770489 2853 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.767158 7891 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.776348 2578 catalog_manager.cc:5484] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.780151 4897 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.792683 3111 consensus_queue.cc:237] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:24.796656 4897 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:24.797008 7905 tablet_replica.cc:330] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:24.797176 5028 raft_consensus.cc:1261] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:24.797251 7905 raft_consensus.cc:2229] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:24.802147 7906 raft_consensus.cc:1050] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:24.803170 7906 consensus_queue.cc:237] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:24.797901 7037 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.007s
I20240502 14:16:24.805244 7891 consensus_queue.cc:1035] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.797494 7905 raft_consensus.cc:2258] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:24.806274 2721 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:24.806840 2983 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:24.807751 5932 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEARNER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:24.807430 7695 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.001s
I20240502 14:16:24.808434 7695 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.808478 7894 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.808920 5028 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.809242 4897 raft_consensus.cc:2932] T 9fd6780b78f849198bc91ed9ae2d08be P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.812100 7695 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.814239 7905 ts_tablet_manager.cc:1875] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:24.814612 2578 catalog_manager.cc:5484] T 9fd6780b78f849198bc91ed9ae2d08be P fa04f9da159541bfbbb6ac55db06e07b reported cstate change: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:24.819818 7905 ts_tablet_manager.cc:1888] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:24.820012 7905 log.cc:1196] T 24027cd85c0d4480bd1cd4639f4f6540 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/24027cd85c0d4480bd1cd4639f4f6540
I20240502 14:16:24.821344 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 24027cd85c0d4480bd1cd4639f4f6540 (table rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]) successfully deleted
I20240502 14:16:24.821722 7202 raft_consensus.cc:1050] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 6648fb2ed9f340ec9bd2d89ad88f2de4 to VOTER
I20240502 14:16:24.822611 7202 consensus_queue.cc:237] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:24.823005 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9fd6780b78f849198bc91ed9ae2d08be with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:24.824446 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 9fd6780b78f849198bc91ed9ae2d08be with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:24.837986 7920 tablet_replica.cc:330] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:24.838191 7273 raft_consensus.cc:1050] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:24.838330 7920 raft_consensus.cc:2229] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:24.838819 7920 raft_consensus.cc:2258] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:24.839814 7920 ts_tablet_manager.cc:1875] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:24.839152 7273 consensus_queue.cc:237] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:24.842609 4633 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.843158 4766 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.843731 7202 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.843820 3111 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.844646 7867 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.844942 7202 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.854091 7695 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 LEADER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.854941 2983 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.859537 7864 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.864480 4634 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.864427 2576 catalog_manager.cc:5484] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.865074 5932 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.865238 7914 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.866222 7914 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.866566 2721 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.866616 2722 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.867619 7920 ts_tablet_manager.cc:1888] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:24.867808 7920 log.cc:1196] T 9fd6780b78f849198bc91ed9ae2d08be P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/9fd6780b78f849198bc91ed9ae2d08be
I20240502 14:16:24.868696 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 9fd6780b78f849198bc91ed9ae2d08be (table rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]) successfully deleted
I20240502 14:16:24.869637 7927 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.870028 3111 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.872671 5932 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.881150 4632 consensus_queue.cc:237] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:24.884109 2722 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:24.884802 7910 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:24.887723 2578 catalog_manager.cc:5484] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.889672 4766 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.896476 5931 raft_consensus.cc:1261] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Refusing update from remote peer 6648fb2ed9f340ec9bd2d89ad88f2de4: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:24.900372 7273 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.900990 4634 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.902210 7695 consensus_queue.cc:1035] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:24.905459 2721 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.905757 7910 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.912134 2721 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.912134 5932 raft_consensus.cc:2932] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
W20240502 14:16:24.912431 2789 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.137:45251, user_credentials={real_user=slave}} blocked reactor thread for 46692.7us
I20240502 14:16:24.914146 2578 catalog_manager.cc:5484] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.918097 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4f3030ae5dc2425586db21b6667e2deb with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:24.917476 4634 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.918929 2577 catalog_manager.cc:5484] T 4f3030ae5dc2425586db21b6667e2deb P 6fb8f2ad13f24e81935b7df7faa291fe reported cstate change: config changed from index 9 to 10, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 2 leader_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.921833 2983 consensus_queue.cc:237] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:24.924789 4766 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:24.925693 7865 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.928416 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 4f3030ae5dc2425586db21b6667e2deb with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:24.935456 7867 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.938043 4633 raft_consensus.cc:1261] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:24.938901 7864 consensus_queue.cc:1035] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:24.938761 5932 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.943751 2565 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 02adec8c6c704326afbedb70578df917 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:24.939636 4766 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.947873 2576 catalog_manager.cc:5484] T 02adec8c6c704326afbedb70578df917 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:24.950660 4633 raft_consensus.cc:2932] T 02adec8c6c704326afbedb70578df917 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } }
I20240502 14:16:24.956881 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 02adec8c6c704326afbedb70578df917 with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:24.957369 7905 tablet_replica.cc:330] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:24.957700 7905 raft_consensus.cc:2229] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:24.958010 7905 raft_consensus.cc:2258] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:24.960871 7905 ts_tablet_manager.cc:1875] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:24.964576 7933 tablet_replica.cc:330] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:24.964879 7933 raft_consensus.cc:2229] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:24.965227 7933 raft_consensus.cc:2258] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:24.966616 7933 ts_tablet_manager.cc:1875] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:24.970831 7905 ts_tablet_manager.cc:1888] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:24.971076 7905 log.cc:1196] T 02adec8c6c704326afbedb70578df917 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/02adec8c6c704326afbedb70578df917
I20240502 14:16:24.972733 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 02adec8c6c704326afbedb70578df917 (table rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]) successfully deleted
I20240502 14:16:24.975061 7933 ts_tablet_manager.cc:1888] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 14:16:24.975173 7933 log.cc:1196] T 4f3030ae5dc2425586db21b6667e2deb P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/4f3030ae5dc2425586db21b6667e2deb
I20240502 14:16:24.975940 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 4f3030ae5dc2425586db21b6667e2deb (table rebalance_test_table_4 [id=49cbf75a88344a85b2f4e552bbfadff7]) successfully deleted
I20240502 14:16:24.984783 7926 raft_consensus.cc:1050] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:24.985622 7926 consensus_queue.cc:237] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:24.988284 4897 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.988627 3111 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.988955 7927 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.989297 7914 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.990037 5932 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:24.991830 7926 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:24.994150 7927 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.995126 3111 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.998318 5932 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:24.998610 4896 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.001785 2576 catalog_manager.cc:5484] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.009184 2851 consensus_queue.cc:237] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:25.011112 5932 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.011193 4896 raft_consensus.cc:1261] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.011715 7931 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.012128 7926 consensus_queue.cc:1035] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.015916 7926 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.019291 5932 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.019459 4897 raft_consensus.cc:2932] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.021579 2568 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:25.024185 2572 catalog_manager.cc:5484] T 56aa6c21e5e947baaeaa6b42c89dfb8f P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:25.028568 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 56aa6c21e5e947baaeaa6b42c89dfb8f with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:25.029268 7905 tablet_replica.cc:330] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:25.029500 7905 raft_consensus.cc:2229] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:25.029909 7905 raft_consensus.cc:2258] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:25.031023 7905 ts_tablet_manager.cc:1875] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:25.037690 7905 ts_tablet_manager.cc:1888] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:25.037832 7905 log.cc:1196] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/56aa6c21e5e947baaeaa6b42c89dfb8f
I20240502 14:16:25.038736 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 56aa6c21e5e947baaeaa6b42c89dfb8f (table rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]) successfully deleted
I20240502 14:16:25.071911 7537 raft_consensus.cc:1050] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:25.072888 7537 consensus_queue.cc:237] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.073230 7876 raft_consensus.cc:1050] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:25.074112 7876 consensus_queue.cc:237] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.078024 5028 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.078095 2983 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.079005 6148 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.079342 4897 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.079721 6148 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.080346 6148 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.100327 7878 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.106518 2983 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.110478 4896 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.116492 4633 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.120545 6148 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.003s
I20240502 14:16:25.121251 2572 catalog_manager.cc:5484] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.123065 7876 raft_consensus.cc:1050] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:25.123950 7876 consensus_queue.cc:237] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:25.126452 5029 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.127079 5028 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.127524 3111 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.128222 7537 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.129640 7971 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.141230 2983 raft_consensus.cc:1261] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:25.142189 4896 raft_consensus.cc:1261] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:25.143404 2851 raft_consensus.cc:1261] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:16:25.144587 7876 raft_consensus.cc:1050] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:25.143153 7537 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.002s
I20240502 14:16:25.145264 7970 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:25.145767 7876 consensus_queue.cc:237] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:25.146445 7971 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:16:25.157447 7878 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.158354 5932 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.158228 3111 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.161540 7971 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.163177 4634 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.164104 2982 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.164768 7537 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.165143 6148 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.165227 5028 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.173038 7970 raft_consensus.cc:1050] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:25.173609 2577 catalog_manager.cc:5484] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:25.174101 7970 consensus_queue.cc:237] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:25.174822 4634 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.178431 6148 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.180790 2981 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.182839 2721 consensus_queue.cc:237] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.186786 4896 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.187327 5029 raft_consensus.cc:1261] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.188253 7971 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.189010 7971 consensus_queue.cc:1035] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.191588 4897 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.193876 5931 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.194638 7878 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.197443 2578 catalog_manager.cc:5484] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.199712 2721 consensus_queue.cc:237] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.200878 6148 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.202379 4634 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.203011 5028 raft_consensus.cc:1261] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.202891 2851 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.206243 7878 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.206610 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet aa0b526e216f46f0a2fb95fe0be02c46 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:25.206614 7971 consensus_queue.cc:1035] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.208492 4896 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.211447 7876 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.214787 2578 catalog_manager.cc:5484] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 8 to 9, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.215118 4634 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.215801 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 652af8d083784c078c5b45cca32ece4b with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:25.216351 5028 raft_consensus.cc:2932] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.220194 5029 raft_consensus.cc:2932] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.228741 7891 raft_consensus.cc:1050] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER fa04f9da159541bfbbb6ac55db06e07b to VOTER
I20240502 14:16:25.228768 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet aa0b526e216f46f0a2fb95fe0be02c46 with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:25.229231 7933 tablet_replica.cc:330] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:25.229503 7933 raft_consensus.cc:2229] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:25.229789 7933 raft_consensus.cc:2258] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:25.229672 7891 consensus_queue.cc:237] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.230142 2982 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.230710 7970 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.230913 7933 ts_tablet_manager.cc:1875] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:25.231617 6148 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.234938 4896 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.236423 2572 catalog_manager.cc:5484] T 652af8d083784c078c5b45cca32ece4b P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 8 to 9, VOTER 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:25.238667 7933 ts_tablet_manager.cc:1888] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:25.238844 7933 log.cc:1196] T aa0b526e216f46f0a2fb95fe0be02c46 P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/aa0b526e216f46f0a2fb95fe0be02c46
I20240502 14:16:25.239048 2722 consensus_queue.cc:237] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:25.239777 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet aa0b526e216f46f0a2fb95fe0be02c46 (table rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]) successfully deleted
I20240502 14:16:25.241480 2851 raft_consensus.cc:1261] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:25.242074 7537 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:25.243147 7878 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.245842 3091 tablet_service.cc:1514] Processing DeleteTablet for tablet 652af8d083784c078c5b45cca32ece4b with delete_type TABLET_DATA_TOMBSTONED (TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:56342
I20240502 14:16:25.246172 7905 tablet_replica.cc:330] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: stopping tablet replica
I20240502 14:16:25.246359 7905 raft_consensus.cc:2229] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:25.246568 7905 raft_consensus.cc:2258] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:25.249243 4633 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.249941 4896 raft_consensus.cc:1261] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:16:25.252903 7905 ts_tablet_manager.cc:1875] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:25.255402 5028 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.255982 7894 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.256142 2578 catalog_manager.cc:5484] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.258251 4634 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.258276 2853 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.258864 7037 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.259294 7878 consensus_queue.cc:1035] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:16:25.259452 7037 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.260753 2851 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.261438 5932 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.263736 7905 ts_tablet_manager.cc:1888] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:25.265669 2983 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.269447 7905 log.cc:1196] T 652af8d083784c078c5b45cca32ece4b P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/652af8d083784c078c5b45cca32ece4b
I20240502 14:16:25.272285 2567 catalog_manager.cc:4830] TS 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): tablet 652af8d083784c078c5b45cca32ece4b (table rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]) successfully deleted
I20240502 14:16:25.275727 4897 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.260093 7876 raft_consensus.cc:2932] T 6f785b6443b741a483e16eb154439b27 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.278815 4896 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.282646 2575 catalog_manager.cc:5484] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 5 to 6, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:25.277155 7989 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.293304 7037 raft_consensus.cc:1050] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 763dcf9158e648b496b4faa1a0dc54af to VOTER
I20240502 14:16:25.294100 7037 consensus_queue.cc:237] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:25.295239 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 6f785b6443b741a483e16eb154439b27 with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:25.295562 7933 tablet_replica.cc:330] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:25.295751 7933 raft_consensus.cc:2229] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:25.295995 7933 raft_consensus.cc:2258] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:25.296587 2580 catalog_manager.cc:5484] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.298236 7933 ts_tablet_manager.cc:1875] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:25.306710 2852 consensus_queue.cc:237] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:25.310442 5932 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.311031 7914 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.310825 5931 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.313692 7914 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.315868 7933 ts_tablet_manager.cc:1888] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:16:25.316045 7933 log.cc:1196] T 6f785b6443b741a483e16eb154439b27 P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/6f785b6443b741a483e16eb154439b27
I20240502 14:16:25.316872 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 6f785b6443b741a483e16eb154439b27 (table rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]) successfully deleted
I20240502 14:16:25.317572 4634 raft_consensus.cc:1261] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.318233 2568 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e4f8998b30c34530a308f78e61138315 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:25.317698 5931 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.320664 2575 catalog_manager.cc:5484] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:25.327816 2721 consensus_queue.cc:237] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:25.328933 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet e4f8998b30c34530a308f78e61138315 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:25.330075 5931 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.286693 2982 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.330684 7970 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.338093 4766 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.338434 2852 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.339133 7894 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.339705 7894 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.340406 7970 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.345608 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 81be921c89194d3bb9c3d41be938ed74 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
W20240502 14:16:25.345705 3047 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.136:39149, user_credentials={real_user=slave}} blocked reactor thread for 48802.5us
I20240502 14:16:25.346992 5931 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.348460 5028 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:25.348271 2576 catalog_manager.cc:5484] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 8 to 9, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:25.349444 7894 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.352732 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 81be921c89194d3bb9c3d41be938ed74 with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:25.353504 7933 tablet_replica.cc:330] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:25.330444 4633 raft_consensus.cc:1261] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.353858 7933 raft_consensus.cc:2229] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:25.354367 7933 raft_consensus.cc:2258] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:25.354456 7914 consensus_queue.cc:1035] T e4f8998b30c34530a308f78e61138315 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.354471 7970 consensus_queue.cc:1035] T 81be921c89194d3bb9c3d41be938ed74 P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.356122 4633 raft_consensus.cc:2932] T 81be921c89194d3bb9c3d41be938ed74 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.357120 4632 raft_consensus.cc:2932] T e4f8998b30c34530a308f78e61138315 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:25.364033 7933 ts_tablet_manager.cc:1875] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:25.374022 7037 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:25.375286 8000 tablet_replica.cc:330] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:25.375767 8000 raft_consensus.cc:2229] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:25.375485 5029 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:25.378492 7891 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.379709 2853 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.382899 5027 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.383965 2578 catalog_manager.cc:5484] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:25.384193 4634 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.386291 8000 raft_consensus.cc:2258] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:25.389567 3111 consensus_queue.cc:237] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:25.389853 2578 catalog_manager.cc:5484] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: config changed from index 7 to 8, fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.388948 4766 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:25.382041 2851 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:25.393143 5028 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.393785 7037 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.396063 4765 raft_consensus.cc:1261] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.397303 7037 consensus_queue.cc:1035] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.397392 3111 consensus_queue.cc:237] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.399644 5027 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.399708 8000 ts_tablet_manager.cc:1875] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:25.399760 4634 raft_consensus.cc:1261] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:25.400184 7999 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.400699 8001 consensus_queue.cc:1035] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:25.401506 4765 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:25.401069 7999 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:25.405265 7891 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.406224 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6f785b6443b741a483e16eb154439b27 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:25.408454 5029 raft_consensus.cc:2932] T 7f7589fa40dd4af09cc87e6e793bcd8a P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:25.410060 2578 catalog_manager.cc:5484] T 7f7589fa40dd4af09cc87e6e793bcd8a P 763dcf9158e648b496b4faa1a0dc54af reported cstate change: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } }
I20240502 14:16:25.411331 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7f7589fa40dd4af09cc87e6e793bcd8a with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:25.411612 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 28f1bc6021e04a05916d3e49a8c98597 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:25.414186 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 7f7589fa40dd4af09cc87e6e793bcd8a with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:25.415812 8005 tablet_replica.cc:330] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:25.416028 8005 raft_consensus.cc:2229] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:25.416289 8005 raft_consensus.cc:2258] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:25.417418 8005 ts_tablet_manager.cc:1875] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:25.418292 7933 ts_tablet_manager.cc:1888] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:25.418448 7933 log.cc:1196] T 81be921c89194d3bb9c3d41be938ed74 P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/81be921c89194d3bb9c3d41be938ed74
I20240502 14:16:25.419176 2577 catalog_manager.cc:5484] T 28f1bc6021e04a05916d3e49a8c98597 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:25.419575 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 81be921c89194d3bb9c3d41be938ed74 (table rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]) successfully deleted
I20240502 14:16:25.419997 8000 ts_tablet_manager.cc:1888] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:25.420177 8000 log.cc:1196] T e4f8998b30c34530a308f78e61138315 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/e4f8998b30c34530a308f78e61138315
I20240502 14:16:25.422602 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet e4f8998b30c34530a308f78e61138315 (table rebalance_test_table_3 [id=7ba8f6c3b48148acb99d80f6786da05b]) successfully deleted
I20240502 14:16:25.423064 4634 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.423739 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 28f1bc6021e04a05916d3e49a8c98597 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:25.427242 5029 raft_consensus.cc:2932] T 28f1bc6021e04a05916d3e49a8c98597 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:25.433400 8005 ts_tablet_manager.cc:1888] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:25.433583 8005 log.cc:1196] T 7f7589fa40dd4af09cc87e6e793bcd8a P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/7f7589fa40dd4af09cc87e6e793bcd8a
I20240502 14:16:25.434513 8005 tablet_replica.cc:330] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:25.434536 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 7f7589fa40dd4af09cc87e6e793bcd8a (table rebalance_test_table_5 [id=fb46cde1c4bc45b0a310d226bfe4da70]) successfully deleted
I20240502 14:16:25.434732 8005 raft_consensus.cc:2229] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:25.435217 8005 raft_consensus.cc:2258] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:25.436333 8005 ts_tablet_manager.cc:1875] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:25.442750 8005 ts_tablet_manager.cc:1888] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:25.442905 8005 log.cc:1196] T 28f1bc6021e04a05916d3e49a8c98597 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/28f1bc6021e04a05916d3e49a8c98597
I20240502 14:16:25.443675 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 28f1bc6021e04a05916d3e49a8c98597 (table rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]) successfully deleted
I20240502 14:16:25.476934 2721 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771"
dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:48350
I20240502 14:16:25.477136 2721 raft_consensus.cc:595] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:25.509274 2721 consensus_queue.cc:237] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 2.7, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:25.511744 4634 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 14:16:25.511798 2851 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
W20240502 14:16:25.511914 2658 consensus_peers.cc:485] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 145a6a19b8474746a30ad51563419f1c. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:25.512517 6148 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.512840 7876 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:25.515878 6148 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 2 LEADER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.516978 7970 raft_consensus.cc:979] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: : Instructing follower 21e5a60e02fd475bb47e5b3a2f294333 to start an election
I20240502 14:16:25.516598 4634 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.517141 7970 raft_consensus.cc:1067] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Signalling peer 21e5a60e02fd475bb47e5b3a2f294333 to start an election
I20240502 14:16:25.517966 4896 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771"
dest_uuid: "21e5a60e02fd475bb47e5b3a2f294333"
from {username='slave'} at 127.27.220.129:40419
I20240502 14:16:25.518147 4896 raft_consensus.cc:482] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 14:16:25.518268 4896 raft_consensus.cc:3037] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:25.517805 2851 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.521049 4896 raft_consensus.cc:504] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.524605 2576 catalog_manager.cc:5484] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 reported cstate change: config changed from index 7 to 8, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 2 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.526873 4896 leader_election.cc:288] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: Requested vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149)
I20240502 14:16:25.527284 2851 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "c6abc8535fe24d03bf71fb7092072fc4"
I20240502 14:16:25.527503 2851 raft_consensus.cc:3037] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:25.528685 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:25.528956 2721 raft_consensus.cc:3032] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:16:25.529063 2721 raft_consensus.cc:729] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:25.529394 2721 consensus_queue.cc:260] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.530128 2721 raft_consensus.cc:3037] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:25.531512 2851 raft_consensus.cc:2454] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:25.536844 2721 raft_consensus.cc:2454] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:25.545653 4832 leader_election.cc:302] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 21e5a60e02fd475bb47e5b3a2f294333, 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4; no voters:
I20240502 14:16:25.546342 6690 raft_consensus.cc:2781] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:25.550191 3111 consensus_queue.cc:237] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:25.554189 4896 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:25.554256 2851 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:25.554827 8001 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:25.555421 7037 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:25.560321 6690 raft_consensus.cc:686] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 LEADER]: Becoming Leader. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Running, Role: LEADER
I20240502 14:16:25.560992 6690 consensus_queue.cc:237] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:25.563004 5029 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cdaeb3ec3d344b4d9ac520f3d4c6d771" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "fa04f9da159541bfbbb6ac55db06e07b"
I20240502 14:16:25.563457 5029 raft_consensus.cc:3037] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:25.567471 5029 raft_consensus.cc:2454] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:25.568465 8001 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.569170 4896 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.571097 2578 catalog_manager.cc:5484] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: term changed from 1 to 2, leader changed from 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) to 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135). New cstate: current_term: 2 leader_uuid: "21e5a60e02fd475bb47e5b3a2f294333" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:25.576161 2576 catalog_manager.cc:5484] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 14:16:25.580932 3049 consensus_peers.cc:485] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6cdd66f946db45478a93459ba69702b2. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:25.585073 2853 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.642089 2721 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550"
dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:48398
I20240502 14:16:25.642318 2721 raft_consensus.cc:595] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:25.659343 7537 raft_consensus.cc:979] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: : Instructing follower 21e5a60e02fd475bb47e5b3a2f294333 to start an election
I20240502 14:16:25.659564 7537 raft_consensus.cc:1067] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Signalling peer 21e5a60e02fd475bb47e5b3a2f294333 to start an election
I20240502 14:16:25.660315 4896 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550"
dest_uuid: "21e5a60e02fd475bb47e5b3a2f294333"
from {username='slave'} at 127.27.220.129:40419
I20240502 14:16:25.660559 4896 raft_consensus.cc:482] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 14:16:25.660651 4896 raft_consensus.cc:3037] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:25.663627 4896 raft_consensus.cc:504] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:25.665225 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:25.665498 2982 raft_consensus.cc:3037] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:25.667215 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:25.667425 2721 raft_consensus.cc:3032] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:16:25.667492 2721 raft_consensus.cc:729] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:25.667743 2721 consensus_queue.cc:260] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:25.668203 2721 raft_consensus.cc:3037] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:25.668362 2982 raft_consensus.cc:2454] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:25.672770 2721 raft_consensus.cc:2454] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:25.673256 4832 leader_election.cc:302] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 21e5a60e02fd475bb47e5b3a2f294333, 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:25.673820 8052 raft_consensus.cc:2781] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:25.674023 8052 raft_consensus.cc:686] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 LEADER]: Becoming Leader. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Running, Role: LEADER
I20240502 14:16:25.674644 4896 leader_election.cc:288] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251)
I20240502 14:16:25.674477 8052 consensus_queue.cc:237] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:25.679656 2578 catalog_manager.cc:5484] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: term changed from 1 to 2, leader changed from 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) to 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135). New cstate: current_term: 2 leader_uuid: "21e5a60e02fd475bb47e5b3a2f294333" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:25.680641 5931 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e882402ebdb24b03a9c51d74dcfd3550" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "6fb8f2ad13f24e81935b7df7faa291fe"
I20240502 14:16:25.680970 5931 raft_consensus.cc:3037] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:25.684178 5931 raft_consensus.cc:2454] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:25.729861 2853 consensus_queue.cc:237] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:25.732453 2721 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:25.733008 4765 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:25.733045 7927 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:25.733791 7927 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 14:16:25.734480 2789 consensus_peers.cc:485] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e9b6901a9a524fd2ab1adb9d730fcd18. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:25.737618 7926 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.742524 4765 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.755369 2721 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.763816 2578 catalog_manager.cc:5484] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af reported cstate change: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.785173 3111 consensus_queue.cc:237] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:25.787575 4765 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:25.787863 2721 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:25.788152 7037 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 14:16:25.788697 3049 consensus_peers.cc:485] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 609b87df71e64919a29a4fa093dfca23. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:25.788697 7037 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:25.791419 7891 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.792066 4765 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.793805 2721 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:25.797228 2576 catalog_manager.cc:5484] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:26.107939 2721 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:26.108819 6690 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:26.114861 2853 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:26.115502 8052 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:26.116533 5931 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:26.117209 8052 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:26.123540 8187 ts_tablet_manager.cc:902] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:26.150741 8194 ts_tablet_manager.cc:902] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:26.167898 5029 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:26.168767 8187 tablet_copy_client.cc:308] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:26.169798 6690 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:26.169979 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 145a6a19b8474746a30ad51563419f1c from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:42751)
I20240502 14:16:26.170138 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 145a6a19b8474746a30ad51563419f1c from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:42751: session id = 6fb8f2ad13f24e81935b7df7faa291fe-145a6a19b8474746a30ad51563419f1c
I20240502 14:16:26.174369 2742 tablet_copy_source_session.cc:205] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:26.176124 8187 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 145a6a19b8474746a30ad51563419f1c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:26.178553 2722 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:26.179318 6690 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:26.180742 2982 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:26.184312 8200 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:26.201817 8187 tablet_copy_client.cc:783] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:26.202123 8187 tablet_copy_client.cc:647] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:26.204979 8199 tablet_copy_client.cc:1002] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:26.243144 8204 ts_tablet_manager.cc:902] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265)
I20240502 14:16:26.248690 8194 tablet_copy_client.cc:308] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:26.250452 4896 consensus_queue.cc:237] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:26.256800 5931 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:26.257593 8052 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:26.259613 2982 raft_consensus.cc:1261] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:26.260455 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 6cdd66f946db45478a93459ba69702b2 from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:56195)
I20240502 14:16:26.260628 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 6cdd66f946db45478a93459ba69702b2 from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:56195: session id = 6fb8f2ad13f24e81935b7df7faa291fe-6cdd66f946db45478a93459ba69702b2
I20240502 14:16:26.262698 6690 consensus_queue.cc:1035] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:26.263631 3133 tablet_copy_source_session.cc:205] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:26.263721 6690 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:26.265223 8194 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cdd66f946db45478a93459ba69702b2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:26.267688 5931 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:26.268142 2982 raft_consensus.cc:2932] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:26.274600 8194 tablet_copy_client.cc:783] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:26.274873 8194 tablet_copy_client.cc:647] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:26.275779 2577 catalog_manager.cc:5484] T e882402ebdb24b03a9c51d74dcfd3550 P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 2 leader_uuid: "21e5a60e02fd475bb47e5b3a2f294333" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:26.276559 8213 tablet_copy_client.cc:1002] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:26.280529 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet e882402ebdb24b03a9c51d74dcfd3550 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:26.282855 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e882402ebdb24b03a9c51d74dcfd3550 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:26.284586 8218 tablet_replica.cc:330] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:26.285005 8218 raft_consensus.cc:2229] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:26.285342 8218 raft_consensus.cc:2258] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:26.285887 8204 tablet_copy_client.cc:308] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:42265
I20240502 14:16:26.286523 8218 ts_tablet_manager.cc:1875] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:26.286672 3133 tablet_copy_service.cc:140] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Received BeginTabletCopySession request for tablet 609b87df71e64919a29a4fa093dfca23 from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:56195)
I20240502 14:16:26.286847 3133 tablet_copy_service.cc:161] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Beginning new tablet copy session on tablet 609b87df71e64919a29a4fa093dfca23 from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:56195: session id = 6fb8f2ad13f24e81935b7df7faa291fe-609b87df71e64919a29a4fa093dfca23
I20240502 14:16:26.289039 3133 tablet_copy_source_session.cc:205] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:26.291009 8204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 609b87df71e64919a29a4fa093dfca23. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:26.293934 8218 ts_tablet_manager.cc:1888] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 14:16:26.294049 8218 log.cc:1196] T e882402ebdb24b03a9c51d74dcfd3550 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/e882402ebdb24b03a9c51d74dcfd3550
I20240502 14:16:26.294833 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet e882402ebdb24b03a9c51d74dcfd3550 (table rebalance_test_table_6 [id=d847769c466e4ee5adf06e8a2ad83846]) successfully deleted
I20240502 14:16:26.296140 8204 tablet_copy_client.cc:783] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:26.296378 8204 tablet_copy_client.cc:647] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:26.297848 8219 tablet_copy_client.cc:1002] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:26.332233 8220 ts_tablet_manager.cc:902] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
I20240502 14:16:26.333817 8220 tablet_copy_client.cc:308] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:44843
I20240502 14:16:26.334558 2873 tablet_copy_service.cc:140] P c6abc8535fe24d03bf71fb7092072fc4: Received BeginTabletCopySession request for tablet e9b6901a9a524fd2ab1adb9d730fcd18 from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:47391)
I20240502 14:16:26.334712 2873 tablet_copy_service.cc:161] P c6abc8535fe24d03bf71fb7092072fc4: Beginning new tablet copy session on tablet e9b6901a9a524fd2ab1adb9d730fcd18 from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:47391: session id = 6fb8f2ad13f24e81935b7df7faa291fe-e9b6901a9a524fd2ab1adb9d730fcd18
I20240502 14:16:26.336999 2873 tablet_copy_source_session.cc:205] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:26.338196 8220 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e9b6901a9a524fd2ab1adb9d730fcd18. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:26.342746 8220 tablet_copy_client.cc:783] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:26.342968 8220 tablet_copy_client.cc:647] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:26.344306 8224 tablet_copy_client.cc:1002] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:26.469655 2721 consensus_queue.cc:237] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
W20240502 14:16:26.472317 2658 consensus_peers.cc:485] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 -> Peer 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137:45251): Couldn't send request to peer 6fb8f2ad13f24e81935b7df7faa291fe. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4aa8dd00e4bb46e2bd8ec2888a54d46b. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:26.472410 4765 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:26.472491 2982 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:26.473102 8261 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:26.473548 7971 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:26.476338 7971 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:26.476862 2982 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:26.483291 2577 catalog_manager.cc:5484] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:26.493981 4765 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } } }
I20240502 14:16:26.730604 4896 consensus_queue.cc:237] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:26.732493 2853 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:26.732477 5029 raft_consensus.cc:1261] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 2 FOLLOWER]: Refusing update from remote peer 21e5a60e02fd475bb47e5b3a2f294333: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:26.732981 6690 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:26.733290 8200 consensus_queue.cc:1035] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:26.738647 6690 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:26.742499 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:26.742637 5029 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P fa04f9da159541bfbbb6ac55db06e07b [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:26.742687 2853 raft_consensus.cc:2932] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } }
I20240502 14:16:26.747962 2577 catalog_manager.cc:5484] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 2 leader_uuid: "21e5a60e02fd475bb47e5b3a2f294333" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:26.754364 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:26.754768 8218 tablet_replica.cc:330] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:26.754988 8218 raft_consensus.cc:2229] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:26.755190 8218 raft_consensus.cc:2258] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:26.756129 8218 ts_tablet_manager.cc:1875] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:26.764619 8218 ts_tablet_manager.cc:1888] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 14:16:26.764844 8218 log.cc:1196] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/cdaeb3ec3d344b4d9ac520f3d4c6d771
I20240502 14:16:26.765821 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet cdaeb3ec3d344b4d9ac520f3d4c6d771 (table rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]) successfully deleted
I20240502 14:16:26.904021 8355 ts_tablet_manager.cc:902] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: Initiating tablet copy from peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
I20240502 14:16:26.905280 8355 tablet_copy_client.cc:308] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:44787
I20240502 14:16:26.906019 2742 tablet_copy_service.cc:140] P 6b3499bcb84b45a094f7166639704ed4: Received BeginTabletCopySession request for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b from peer 6fb8f2ad13f24e81935b7df7faa291fe ({username='slave'} at 127.27.220.137:42751)
I20240502 14:16:26.906165 2742 tablet_copy_service.cc:161] P 6b3499bcb84b45a094f7166639704ed4: Beginning new tablet copy session on tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b from peer 6fb8f2ad13f24e81935b7df7faa291fe at {username='slave'} at 127.27.220.137:42751: session id = 6fb8f2ad13f24e81935b7df7faa291fe-4aa8dd00e4bb46e2bd8ec2888a54d46b
I20240502 14:16:26.908079 2742 tablet_copy_source_session.cc:205] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:26.909332 8355 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:26.914537 8355 tablet_copy_client.cc:783] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:26.914764 8355 tablet_copy_client.cc:647] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:26.916262 8359 tablet_copy_client.cc:1002] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:27.705417 8187 tablet_copy_client.cc:523] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:27.708832 8187 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:27.743129 8187 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:27.743607 8187 tablet_bootstrap.cc:492] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:27.743908 8187 ts_tablet_manager.cc:1367] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.035s user 0.030s sys 0.004s
I20240502 14:16:27.744534 8187 raft_consensus.cc:348] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:27.744741 8187 raft_consensus.cc:729] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:27.744939 8187 consensus_queue.cc:260] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:27.745694 8187 ts_tablet_manager.cc:1398] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.002s user 0.003s sys 0.001s
I20240502 14:16:27.746433 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-145a6a19b8474746a30ad51563419f1c received from {username='slave'} at 127.27.220.137:42751
I20240502 14:16:27.746614 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-145a6a19b8474746a30ad51563419f1c on tablet 145a6a19b8474746a30ad51563419f1c with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:27.776906 8194 tablet_copy_client.cc:523] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:27.779623 8194 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:27.798229 8204 tablet_copy_client.cc:523] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:27.801092 8204 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:27.812667 8194 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:27.813094 8194 tablet_bootstrap.cc:492] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:27.813387 8194 ts_tablet_manager.cc:1367] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.034s user 0.026s sys 0.006s
I20240502 14:16:27.814036 8194 raft_consensus.cc:348] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:27.814285 8194 raft_consensus.cc:729] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:27.814522 8194 consensus_queue.cc:260] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:27.814996 8194 ts_tablet_manager.cc:1398] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.001s user 0.000s sys 0.004s
I20240502 14:16:27.815834 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-6cdd66f946db45478a93459ba69702b2 received from {username='slave'} at 127.27.220.137:56195
I20240502 14:16:27.815975 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-6cdd66f946db45478a93459ba69702b2 on tablet 6cdd66f946db45478a93459ba69702b2 with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:27.835717 8204 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:27.836289 8204 tablet_bootstrap.cc:492] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:27.836558 8204 ts_tablet_manager.cc:1367] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.036s user 0.036s sys 0.000s
I20240502 14:16:27.837318 8204 raft_consensus.cc:348] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:27.837630 8204 raft_consensus.cc:729] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:27.837846 8204 consensus_queue.cc:260] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:27.838567 8204 ts_tablet_manager.cc:1398] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:16:27.839370 3133 tablet_copy_service.cc:342] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-609b87df71e64919a29a4fa093dfca23 received from {username='slave'} at 127.27.220.137:56195
I20240502 14:16:27.839529 3133 tablet_copy_service.cc:433] P 472a7cc2e2d04cf0b36f7e43ed55f1d2: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-609b87df71e64919a29a4fa093dfca23 on tablet 609b87df71e64919a29a4fa093dfca23 with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:27.844584 8220 tablet_copy_client.cc:523] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:27.849215 8220 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:27.868291 5931 raft_consensus.cc:1203] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEARNER]: Deduplicated request from leader. Original: 2.7->[2.8-2.8] Dedup: 2.8->[]
I20240502 14:16:27.889124 8220 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:27.889631 8220 tablet_bootstrap.cc:492] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:27.889928 8220 ts_tablet_manager.cc:1367] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.041s user 0.039s sys 0.004s
I20240502 14:16:27.890547 8220 raft_consensus.cc:348] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:27.890779 8220 raft_consensus.cc:729] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:27.890996 8220 consensus_queue.cc:260] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:27.891613 8220 ts_tablet_manager.cc:1398] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:27.892654 2873 tablet_copy_service.cc:342] P c6abc8535fe24d03bf71fb7092072fc4: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-e9b6901a9a524fd2ab1adb9d730fcd18 received from {username='slave'} at 127.27.220.137:47391
I20240502 14:16:27.892825 2873 tablet_copy_service.cc:433] P c6abc8535fe24d03bf71fb7092072fc4: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-e9b6901a9a524fd2ab1adb9d730fcd18 on tablet e9b6901a9a524fd2ab1adb9d730fcd18 with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:28.219033 5931 raft_consensus.cc:1203] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:28.269774 5931 raft_consensus.cc:1203] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:28.382139 8415 raft_consensus.cc:1050] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:28.383141 8415 consensus_queue.cc:237] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:28.385780 5931 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:28.386251 2853 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:28.386593 8415 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.386893 4633 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:28.387295 8415 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.389129 5930 raft_consensus.cc:1203] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:28.390098 8472 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.396346 8472 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 2 LEADER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.397217 4633 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.401861 5931 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.402357 2576 catalog_manager.cc:5484] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 reported cstate change: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:28.411451 2853 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.416654 8355 tablet_copy_client.cc:523] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:28.420337 8355 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap starting.
I20240502 14:16:28.441963 2721 consensus_queue.cc:237] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:28.444183 4633 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:28.444191 5931 raft_consensus.cc:1261] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:28.444736 8415 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:28.445487 8415 consensus_queue.cc:1035] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:28.447973 8521 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6b3499bcb84b45a094f7166639704ed4 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.449378 5931 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.452023 4633 raft_consensus.cc:2932] T 145a6a19b8474746a30ad51563419f1c P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.454900 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 145a6a19b8474746a30ad51563419f1c with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:28.454244 2575 catalog_manager.cc:5484] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe reported cstate change: config changed from index 9 to 10, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 2 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.460136 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 145a6a19b8474746a30ad51563419f1c with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:28.460867 8534 tablet_replica.cc:330] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:28.461370 8534 raft_consensus.cc:2229] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:28.461747 8534 raft_consensus.cc:2258] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:28.462805 8534 ts_tablet_manager.cc:1875] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:28.468150 8534 ts_tablet_manager.cc:1888] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 14:16:28.468370 8534 log.cc:1196] T 145a6a19b8474746a30ad51563419f1c P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/145a6a19b8474746a30ad51563419f1c
I20240502 14:16:28.469121 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 145a6a19b8474746a30ad51563419f1c (table rebalance_test_table_8 [id=60273368f9a8485eb20eeec643b2d902]) successfully deleted
I20240502 14:16:28.475306 8355 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:28.475716 8355 tablet_bootstrap.cc:492] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: Bootstrap complete.
I20240502 14:16:28.476019 8355 ts_tablet_manager.cc:1367] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent bootstrapping tablet: real 0.056s user 0.045s sys 0.004s
I20240502 14:16:28.476791 8355 raft_consensus.cc:348] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:28.477020 8355 raft_consensus.cc:729] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Initialized, Role: LEARNER
I20240502 14:16:28.477246 8355 consensus_queue.cc:260] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: true } }
I20240502 14:16:28.477905 8355 ts_tablet_manager.cc:1398] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:16:28.478812 2742 tablet_copy_service.cc:342] P 6b3499bcb84b45a094f7166639704ed4: Request end of tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-4aa8dd00e4bb46e2bd8ec2888a54d46b received from {username='slave'} at 127.27.220.137:42751
I20240502 14:16:28.478976 2742 tablet_copy_service.cc:433] P 6b3499bcb84b45a094f7166639704ed4: ending tablet copy session 6fb8f2ad13f24e81935b7df7faa291fe-4aa8dd00e4bb46e2bd8ec2888a54d46b on tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b with peer 6fb8f2ad13f24e81935b7df7faa291fe
I20240502 14:16:28.647727 8557 raft_consensus.cc:1050] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:28.648605 8557 consensus_queue.cc:237] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:28.651400 5931 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.651490 4896 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.652107 8544 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.652459 8392 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.653973 2853 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.656044 8544 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.658520 8392 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.661278 2853 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.665809 5931 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.670495 2578 catalog_manager.cc:5484] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:28.677117 4896 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.680675 3110 consensus_queue.cc:237] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:28.683913 5931 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:28.684010 4896 raft_consensus.cc:1261] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:28.684783 8392 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.685099 8557 consensus_queue.cc:1035] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.687309 8392 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.687906 4896 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.688378 5930 raft_consensus.cc:2932] T 6cdd66f946db45478a93459ba69702b2 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.691433 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6cdd66f946db45478a93459ba69702b2 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:28.694576 2575 catalog_manager.cc:5484] T 6cdd66f946db45478a93459ba69702b2 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 8 to 9, VOTER c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.699469 2833 tablet_service.cc:1514] Processing DeleteTablet for tablet 6cdd66f946db45478a93459ba69702b2 with delete_type TABLET_DATA_TOMBSTONED (TS c6abc8535fe24d03bf71fb7092072fc4 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:51766
I20240502 14:16:28.700119 8534 tablet_replica.cc:330] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: stopping tablet replica
I20240502 14:16:28.700326 8534 raft_consensus.cc:2229] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:28.700568 8534 raft_consensus.cc:2258] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:28.701567 8534 ts_tablet_manager.cc:1875] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:28.706625 8534 ts_tablet_manager.cc:1888] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:28.706739 8534 log.cc:1196] T 6cdd66f946db45478a93459ba69702b2 P c6abc8535fe24d03bf71fb7092072fc4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/6cdd66f946db45478a93459ba69702b2
I20240502 14:16:28.707453 2568 catalog_manager.cc:4830] TS c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): tablet 6cdd66f946db45478a93459ba69702b2 (table rebalance_test_table_1 [id=635f67e7410241a7ba401da689d97b6c]) successfully deleted
I20240502 14:16:28.885130 8599 raft_consensus.cc:1050] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:28.886083 8599 consensus_queue.cc:237] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:28.889345 4765 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.889374 5931 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.889935 2721 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.890061 8598 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.890512 8601 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.890894 8599 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.897163 8361 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.899575 4765 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.901687 5931 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.902570 2721 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.906919 2577 catalog_manager.cc:5484] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:28.913942 2853 consensus_queue.cc:237] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:28.916009 4765 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:28.916214 8392 raft_consensus.cc:1050] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:28.916353 5930 raft_consensus.cc:1261] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Refusing update from remote peer c6abc8535fe24d03bf71fb7092072fc4: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:28.916903 8601 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.917207 8361 consensus_queue.cc:1035] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.917093 8392 consensus_queue.cc:237] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:28.919618 8361 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.921175 4766 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.923300 2568 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e9b6901a9a524fd2ab1adb9d730fcd18 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:28.925190 2578 catalog_manager.cc:5484] T e9b6901a9a524fd2ab1adb9d730fcd18 P c6abc8535fe24d03bf71fb7092072fc4 reported cstate change: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "c6abc8535fe24d03bf71fb7092072fc4" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:28.928946 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet e9b6901a9a524fd2ab1adb9d730fcd18 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:28.929229 5930 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.929226 2721 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.929874 8608 tablet_replica.cc:330] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:28.929775 8557 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.930130 4765 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:28.930420 8557 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.930713 8392 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:28.930158 8608 raft_consensus.cc:2229] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:28.932411 8608 raft_consensus.cc:2258] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:28.933568 8608 ts_tablet_manager.cc:1875] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:28.933846 5932 raft_consensus.cc:1203] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:28.940117 8557 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.938601 5931 raft_consensus.cc:2932] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.942169 8608 ts_tablet_manager.cc:1888] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:28.942301 8608 log.cc:1196] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/e9b6901a9a524fd2ab1adb9d730fcd18
I20240502 14:16:28.943037 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet e9b6901a9a524fd2ab1adb9d730fcd18 (table rebalance_test_table_0 [id=755ba2c8b92744f0a5c53f7fa7ed6c15]) successfully deleted
I20240502 14:16:28.944905 5930 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.944789 2721 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.946079 2578 catalog_manager.cc:5484] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:28.960582 4765 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.962370 3110 consensus_queue.cc:237] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:28.964336 5931 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:28.964388 4765 raft_consensus.cc:1261] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 472a7cc2e2d04cf0b36f7e43ed55f1d2: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:16:28.964906 8559 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.966272 8544 consensus_queue.cc:1035] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:28.967283 8559 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.967883 5931 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.969216 4765 raft_consensus.cc:2932] T 609b87df71e64919a29a4fa093dfca23 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:28.971197 2567 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 609b87df71e64919a29a4fa093dfca23 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:28.973436 2577 catalog_manager.cc:5484] T 609b87df71e64919a29a4fa093dfca23 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 8 to 9, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:28.977075 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet 609b87df71e64919a29a4fa093dfca23 with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:28.977463 8608 tablet_replica.cc:330] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:28.977703 8608 raft_consensus.cc:2229] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:28.977981 8608 raft_consensus.cc:2258] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:28.978972 8608 ts_tablet_manager.cc:1875] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:28.983862 8608 ts_tablet_manager.cc:1888] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 14:16:28.983968 8608 log.cc:1196] T 609b87df71e64919a29a4fa093dfca23 P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/609b87df71e64919a29a4fa093dfca23
I20240502 14:16:28.984670 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet 609b87df71e64919a29a4fa093dfca23 (table rebalance_test_table_7 [id=5dee5b2132e849b0b40fb5306f717ce6]) successfully deleted
I20240502 14:16:29.318918 8472 raft_consensus.cc:1050] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: attempting to promote NON_VOTER 6fb8f2ad13f24e81935b7df7faa291fe to VOTER
I20240502 14:16:29.319888 8472 consensus_queue.cc:237] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:29.322775 5931 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 LEARNER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:29.323312 8472 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:29.323522 2982 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:29.324046 8637 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:29.324255 4765 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Refusing update from remote peer 6b3499bcb84b45a094f7166639704ed4: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:29.325237 8472 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:29.327519 8637 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:29.328197 2982 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:29.328420 4765 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:29.332826 5931 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:29.333958 2578 catalog_manager.cc:5484] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e reported cstate change: config changed from index 7 to 8, 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6b3499bcb84b45a094f7166639704ed4" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:29.483635 2721 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b"
dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:48654
I20240502 14:16:29.483842 2721 raft_consensus.cc:595] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:29.708297 2721 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b"
dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:48678
I20240502 14:16:29.708518 2721 raft_consensus.cc:595] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:29.730463 8472 raft_consensus.cc:979] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: : Instructing follower 6fb8f2ad13f24e81935b7df7faa291fe to start an election
I20240502 14:16:29.730654 8472 raft_consensus.cc:1067] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Signalling peer 6fb8f2ad13f24e81935b7df7faa291fe to start an election
I20240502 14:16:29.731385 5931 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b"
dest_uuid: "6fb8f2ad13f24e81935b7df7faa291fe"
from {username='slave'} at 127.27.220.129:50107
I20240502 14:16:29.731727 5931 raft_consensus.cc:482] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 14:16:29.731863 5931 raft_consensus.cc:3037] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:29.734645 5931 raft_consensus.cc:504] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:29.736641 2721 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b" candidate_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "6b3499bcb84b45a094f7166639704ed4"
I20240502 14:16:29.736863 2721 raft_consensus.cc:3032] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:16:29.736975 2721 raft_consensus.cc:729] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 6b3499bcb84b45a094f7166639704ed4, State: Running, Role: LEADER
I20240502 14:16:29.737265 2721 consensus_queue.cc:260] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:29.737777 2721 raft_consensus.cc:3037] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:29.739868 2721 raft_consensus.cc:2454] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6fb8f2ad13f24e81935b7df7faa291fe in term 2.
I20240502 14:16:29.741375 5931 leader_election.cc:288] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 election: Requested vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403)
I20240502 14:16:29.749722 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b" candidate_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:29.749981 2982 raft_consensus.cc:3037] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:29.749897 4765 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4aa8dd00e4bb46e2bd8ec2888a54d46b" candidate_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "763dcf9158e648b496b4faa1a0dc54af"
I20240502 14:16:29.750262 4765 raft_consensus.cc:3037] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:29.752866 2982 raft_consensus.cc:2454] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6fb8f2ad13f24e81935b7df7faa291fe in term 2.
I20240502 14:16:29.753307 4765 raft_consensus.cc:2454] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6fb8f2ad13f24e81935b7df7faa291fe in term 2.
I20240502 14:16:29.753787 5863 leader_election.cc:302] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 6b3499bcb84b45a094f7166639704ed4, 6fb8f2ad13f24e81935b7df7faa291fe, 8cee803d912546e1981521fd8f16bb4e; no voters:
I20240502 14:16:29.754197 8442 raft_consensus.cc:2781] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:29.754724 8442 raft_consensus.cc:686] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEADER]: Becoming Leader. State: Replica: 6fb8f2ad13f24e81935b7df7faa291fe, State: Running, Role: LEADER
I20240502 14:16:29.755187 8442 consensus_queue.cc:237] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:29.785240 2576 catalog_manager.cc:5484] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe reported cstate change: term changed from 1 to 2, leader changed from 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) to 6fb8f2ad13f24e81935b7df7faa291fe (127.27.220.137). New cstate: current_term: 2 leader_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:30.207655 4765 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 2 FOLLOWER]: Refusing update from remote peer 6fb8f2ad13f24e81935b7df7faa291fe: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:30.208299 8442 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:30.215734 2721 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Refusing update from remote peer 6fb8f2ad13f24e81935b7df7faa291fe: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:30.216399 8442 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [LEADER]: Connected to new peer: Peer: permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:30.220737 2982 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 6fb8f2ad13f24e81935b7df7faa291fe: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:30.221662 8666 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:30.769248 5931 consensus_queue.cc:237] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:30.771492 4765 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 2 FOLLOWER]: Refusing update from remote peer 6fb8f2ad13f24e81935b7df7faa291fe: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:30.772183 2982 raft_consensus.cc:1261] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer 6fb8f2ad13f24e81935b7df7faa291fe: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:30.772205 8696 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [LEADER]: Connected to new peer: Peer: permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:30.773334 8696 consensus_queue.cc:1035] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:30.775807 8697 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:30.776690 4765 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 763dcf9158e648b496b4faa1a0dc54af [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:30.778302 2982 raft_consensus.cc:2932] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } } }
I20240502 14:16:30.780339 2568 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:30.781985 2575 catalog_manager.cc:5484] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe reported cstate change: config changed from index 8 to 10, VOTER 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) evicted. New cstate: current_term: 2 leader_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:16:30.789366 2701 tablet_service.cc:1514] Processing DeleteTablet for tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b with delete_type TABLET_DATA_TOMBSTONED (TS 6b3499bcb84b45a094f7166639704ed4 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:39782
I20240502 14:16:30.803791 8715 tablet_replica.cc:330] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: stopping tablet replica
I20240502 14:16:30.804016 8715 raft_consensus.cc:2229] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:30.804262 8715 raft_consensus.cc:2258] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:30.805186 8715 ts_tablet_manager.cc:1875] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:30.811350 8715 ts_tablet_manager.cc:1888] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 14:16:30.811491 8715 log.cc:1196] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6b3499bcb84b45a094f7166639704ed4: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/4aa8dd00e4bb46e2bd8ec2888a54d46b
I20240502 14:16:30.812386 2566 catalog_manager.cc:4830] TS 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b (table rebalance_test_table_2 [id=f20db0b50ef747efa3778cefee2ed81c]) successfully deleted
I20240502 14:16:30.928045 3091 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:30.943881 5910 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:30.946768 4878 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:16:30.953696 2701 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:30.958660 4614 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:31.011359 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:31.011426 5009 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:31.030723 2963 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:31.057186 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
W20240502 14:16:31.117627 3047 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.136:39149, user_credentials={real_user=slave}} blocked reactor thread for 63900.3us
I20240502 14:16:31.163936 2982 consensus_queue.cc:237] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:31.166622 5029 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:16:31.166621 3110 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 14:16:31.167232 2917 consensus_peers.cc:485] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e -> Peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Couldn't send request to peer 21e5a60e02fd475bb47e5b3a2f294333. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3cc7001d1b9b425a8758f8e15914c289. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:31.167500 8690 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:31.168200 8748 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:16:31.171388 8690 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:31.172051 5029 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:31.172899 3110 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:31.180895 2575 catalog_manager.cc:5484] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 reported cstate change: config changed from index 6 to 7, NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) added. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } } }
I20240502 14:16:31.774926 8784 ts_tablet_manager.cc:902] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: Initiating tablet copy from peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567)
I20240502 14:16:31.776183 8784 tablet_copy_client.cc:308] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:36567
I20240502 14:16:31.776825 3003 tablet_copy_service.cc:140] P 8cee803d912546e1981521fd8f16bb4e: Received BeginTabletCopySession request for tablet 3cc7001d1b9b425a8758f8e15914c289 from peer 21e5a60e02fd475bb47e5b3a2f294333 ({username='slave'} at 127.27.220.135:37359)
I20240502 14:16:31.776985 3003 tablet_copy_service.cc:161] P 8cee803d912546e1981521fd8f16bb4e: Beginning new tablet copy session on tablet 3cc7001d1b9b425a8758f8e15914c289 from peer 21e5a60e02fd475bb47e5b3a2f294333 at {username='slave'} at 127.27.220.135:37359: session id = 21e5a60e02fd475bb47e5b3a2f294333-3cc7001d1b9b425a8758f8e15914c289
I20240502 14:16:31.778926 3003 tablet_copy_source_session.cc:205] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:16:31.780100 8784 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3cc7001d1b9b425a8758f8e15914c289. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:31.784543 8784 tablet_copy_client.cc:783] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 0 data blocks...
I20240502 14:16:31.784757 8784 tablet_copy_client.cc:647] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Starting download of 1 WAL segments...
I20240502 14:16:31.786222 8788 tablet_copy_client.cc:1002] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Injecting latency into file download: 1500
I20240502 14:16:33.286687 8784 tablet_copy_client.cc:523] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:16:33.289713 8784 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap starting.
I20240502 14:16:33.328445 8784 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:33.345829 8784 tablet_bootstrap.cc:492] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: Bootstrap complete.
I20240502 14:16:33.346165 8784 ts_tablet_manager.cc:1367] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent bootstrapping tablet: real 0.057s user 0.030s sys 0.009s
I20240502 14:16:33.346911 8784 raft_consensus.cc:348] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:33.347157 8784 raft_consensus.cc:729] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Initialized, Role: LEARNER
I20240502 14:16:33.347318 8784 consensus_queue.cc:260] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: NON_VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: true } }
I20240502 14:16:33.347851 8784 ts_tablet_manager.cc:1398] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:16:33.348678 3003 tablet_copy_service.cc:342] P 8cee803d912546e1981521fd8f16bb4e: Request end of tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-3cc7001d1b9b425a8758f8e15914c289 received from {username='slave'} at 127.27.220.135:37359
I20240502 14:16:33.348826 3003 tablet_copy_service.cc:433] P 8cee803d912546e1981521fd8f16bb4e: ending tablet copy session 21e5a60e02fd475bb47e5b3a2f294333-3cc7001d1b9b425a8758f8e15914c289 on tablet 3cc7001d1b9b425a8758f8e15914c289 with peer 21e5a60e02fd475bb47e5b3a2f294333
I20240502 14:16:33.350855 4897 raft_consensus.cc:1203] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 14:16:33.806406 8852 raft_consensus.cc:1050] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: attempting to promote NON_VOTER 21e5a60e02fd475bb47e5b3a2f294333 to VOTER
I20240502 14:16:33.807333 8852 consensus_queue.cc:237] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:33.810448 3110 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:33.810892 4897 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 LEARNER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:33.810897 5028 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Refusing update from remote peer 8cee803d912546e1981521fd8f16bb4e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:16:33.811407 8851 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:33.811743 8802 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:33.812142 8854 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:16:33.815071 8854 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:33.815690 5028 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:33.816468 4897 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:33.819844 3110 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:33.822674 2577 catalog_manager.cc:5484] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b reported cstate change: config changed from index 7 to 8, 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8cee803d912546e1981521fd8f16bb4e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:33.934841 2982 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289"
dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:53938
I20240502 14:16:33.935062 2982 raft_consensus.cc:595] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:34.151692 2982 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289"
dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:53966
I20240502 14:16:34.151894 2982 raft_consensus.cc:595] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Received request to transfer leadership
I20240502 14:16:34.220814 8802 raft_consensus.cc:979] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: : Instructing follower fa04f9da159541bfbbb6ac55db06e07b to start an election
I20240502 14:16:34.220995 8802 raft_consensus.cc:1067] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Signalling peer fa04f9da159541bfbbb6ac55db06e07b to start an election
I20240502 14:16:34.221760 5028 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289"
dest_uuid: "fa04f9da159541bfbbb6ac55db06e07b"
from {username='slave'} at 127.27.220.131:42913
I20240502 14:16:34.221993 5028 raft_consensus.cc:482] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 14:16:34.222131 5028 raft_consensus.cc:3037] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:34.224334 5028 raft_consensus.cc:504] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:34.225912 5028 leader_election.cc:288] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 election: Requested vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265), 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331)
I20240502 14:16:34.226159 2982 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289" candidate_uuid: "fa04f9da159541bfbbb6ac55db06e07b" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "8cee803d912546e1981521fd8f16bb4e"
I20240502 14:16:34.226385 2982 raft_consensus.cc:3032] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Stepping down as leader of term 1
I20240502 14:16:34.226361 3110 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289" candidate_uuid: "fa04f9da159541bfbbb6ac55db06e07b" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2"
I20240502 14:16:34.226476 2982 raft_consensus.cc:729] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 8cee803d912546e1981521fd8f16bb4e, State: Running, Role: LEADER
I20240502 14:16:34.226603 3110 raft_consensus.cc:3037] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:34.226780 2982 consensus_queue.cc:260] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:34.227176 2982 raft_consensus.cc:3037] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:34.229460 3110 raft_consensus.cc:2454] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa04f9da159541bfbbb6ac55db06e07b in term 2.
I20240502 14:16:34.230038 2982 raft_consensus.cc:2454] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa04f9da159541bfbbb6ac55db06e07b in term 2.
I20240502 14:16:34.230494 4962 leader_election.cc:302] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 472a7cc2e2d04cf0b36f7e43ed55f1d2, 8cee803d912546e1981521fd8f16bb4e, fa04f9da159541bfbbb6ac55db06e07b; no voters:
I20240502 14:16:34.230870 8858 raft_consensus.cc:2781] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:34.231411 8858 raft_consensus.cc:686] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 2 LEADER]: Becoming Leader. State: Replica: fa04f9da159541bfbbb6ac55db06e07b, State: Running, Role: LEADER
I20240502 14:16:34.231932 8858 consensus_queue.cc:237] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:34.233487 4897 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3cc7001d1b9b425a8758f8e15914c289" candidate_uuid: "fa04f9da159541bfbbb6ac55db06e07b" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "21e5a60e02fd475bb47e5b3a2f294333"
I20240502 14:16:34.233783 4897 raft_consensus.cc:3037] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:34.235988 4897 raft_consensus.cc:2454] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate fa04f9da159541bfbbb6ac55db06e07b in term 2.
I20240502 14:16:34.235766 2572 catalog_manager.cc:5484] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b reported cstate change: term changed from 1 to 2, leader changed from 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) to fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136). New cstate: current_term: 2 leader_uuid: "fa04f9da159541bfbbb6ac55db06e07b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:34.706771 3110 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Refusing update from remote peer fa04f9da159541bfbbb6ac55db06e07b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:34.707399 8858 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:34.790148 2982 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Refusing update from remote peer fa04f9da159541bfbbb6ac55db06e07b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:34.790695 8858 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [LEADER]: Connected to new peer: Peer: permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:34.795882 4897 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Refusing update from remote peer fa04f9da159541bfbbb6ac55db06e07b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 14:16:34.796447 8907 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:16:35.334887 5028 consensus_queue.cc:237] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:35.336959 4897 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Refusing update from remote peer fa04f9da159541bfbbb6ac55db06e07b: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:35.336975 3110 raft_consensus.cc:1261] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Refusing update from remote peer fa04f9da159541bfbbb6ac55db06e07b: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 14:16:35.337543 8923 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [LEADER]: Connected to new peer: Peer: permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:35.337880 8924 consensus_queue.cc:1035] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [LEADER]: Connected to new peer: Peer: permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:16:35.340088 8923 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:35.340732 3110 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 472a7cc2e2d04cf0b36f7e43ed55f1d2 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:35.341269 4897 raft_consensus.cc:2932] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:35.345257 2566 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3cc7001d1b9b425a8758f8e15914c289 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:16:35.345890 2572 catalog_manager.cc:5484] T 3cc7001d1b9b425a8758f8e15914c289 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: config changed from index 8 to 10, VOTER 8cee803d912546e1981521fd8f16bb4e (127.27.220.131) evicted. New cstate: current_term: 2 leader_uuid: "fa04f9da159541bfbbb6ac55db06e07b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "472a7cc2e2d04cf0b36f7e43ed55f1d2" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 42265 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } }
I20240502 14:16:35.350924 2963 tablet_service.cc:1514] Processing DeleteTablet for tablet 3cc7001d1b9b425a8758f8e15914c289 with delete_type TABLET_DATA_TOMBSTONED (TS 8cee803d912546e1981521fd8f16bb4e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:42890
I20240502 14:16:35.351588 8930 tablet_replica.cc:330] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: stopping tablet replica
I20240502 14:16:35.351840 8930 raft_consensus.cc:2229] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 14:16:35.352126 8930 raft_consensus.cc:2258] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 14:16:35.353104 8930 ts_tablet_manager.cc:1875] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:16:35.357856 8930 ts_tablet_manager.cc:1888] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 14:16:35.357959 8930 log.cc:1196] T 3cc7001d1b9b425a8758f8e15914c289 P 8cee803d912546e1981521fd8f16bb4e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/3cc7001d1b9b425a8758f8e15914c289
I20240502 14:16:35.358675 2565 catalog_manager.cc:4830] TS 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): tablet 3cc7001d1b9b425a8758f8e15914c289 (table rebalance_test_table_9 [id=3af4db3a3ff148b281028d85f9355fd6]) successfully deleted
I20240502 14:16:35.574739 4614 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:35.578399 3091 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:35.602874 2701 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:35.627564 4878 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:16:35.652127 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:35.655933 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:35.665197 5009 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:35.679623 5910 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:35.682674 2963 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:35.910759 4878 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:16:35.936556 4614 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:35.937227 3091 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:35.957568 5910 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:35.959643 2701 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:35.972699 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:35.974017 5009 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:35.974404 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:35.983810 2963 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:36.954697 5910 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:36.963021 2963 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:36.964396 4878 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:16:36.977061 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:36.993275 3091 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:37.002048 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:37.011804 2701 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:37.013442 5009 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:37.026495 4614 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:37.283018 3091 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:37.303521 5910 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:37.326557 4878 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:16:37.331743 4614 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:37.338182 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:37.342811 2701 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:37.374377 2963 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:37.398512 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:37.419355 5009 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:37.620186 4878 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:16:37.637557 3091 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:37.663110 4614 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:37.681814 2963 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:37.685134 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:37.690902 2701 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:37.710927 5910 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:37.738350 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:37.740629 5009 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:38.095392 4878 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:16:38.109737 3091 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:38.117760 4614 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:38.133503 2701 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 14:16:38.134248 5910 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:38.220510 4746 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:38.224417 2833 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:16:38.227536 2963 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:38.235183 5009 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
5f080ba1a0a44cb0982d5c0d67b8633f | 127.27.220.190:35583 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+----------------------+-------------------------
builtin_ntp_servers | 127.27.220.148:43555 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+----------------------+---------+----------+----------------+-----------------
21e5a60e02fd475bb47e5b3a2f294333 | 127.27.220.135:35331 | HEALTHY | <none> | 3 | 0
472a7cc2e2d04cf0b36f7e43ed55f1d2 | 127.27.220.132:42265 | HEALTHY | <none> | 7 | 0
6648fb2ed9f340ec9bd2d89ad88f2de4 | 127.27.220.133:41841 | HEALTHY | <none> | 1 | 0
6b3499bcb84b45a094f7166639704ed4 | 127.27.220.129:44787 | HEALTHY | <none> | 7 | 0
6fb8f2ad13f24e81935b7df7faa291fe | 127.27.220.137:45251 | HEALTHY | <none> | 1 | 0
763dcf9158e648b496b4faa1a0dc54af | 127.27.220.134:41403 | HEALTHY | <none> | 0 | 0
8cee803d912546e1981521fd8f16bb4e | 127.27.220.131:36567 | HEALTHY | <none> | 6 | 0
c6abc8535fe24d03bf71fb7092072fc4 | 127.27.220.130:44843 | HEALTHY | <none> | 4 | 0
fa04f9da159541bfbbb6ac55db06e07b | 127.27.220.136:39149 | HEALTHY | <none> | 1 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 9
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
---------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 9 server(s) checked
local_ip_for_outbound_sockets | 127.27.220.129 | experimental | 127.27.220.129:44787
local_ip_for_outbound_sockets | 127.27.220.130 | experimental | 127.27.220.130:44843
local_ip_for_outbound_sockets | 127.27.220.131 | experimental | 127.27.220.131:36567
local_ip_for_outbound_sockets | 127.27.220.132 | experimental | 127.27.220.132:42265
local_ip_for_outbound_sockets | 127.27.220.133 | experimental | 127.27.220.133:41841
local_ip_for_outbound_sockets | 127.27.220.134 | experimental | 127.27.220.134:41403
local_ip_for_outbound_sockets | 127.27.220.135 | experimental | 127.27.220.135:35331
local_ip_for_outbound_sockets | 127.27.220.136 | experimental | 127.27.220.136:39149
local_ip_for_outbound_sockets | 127.27.220.137 | experimental | 127.27.220.137:45251
never_fsync | true | unsafe,advanced | all 9 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 9 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 9 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 9 server(s) checked
server_dump_info_format | pb | hidden | all 9 server(s) checked
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.27.220.129:44787
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.27.220.130:44843
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.27.220.131:36567
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.27.220.132:42265
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.27.220.133:41841
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.27.220.134:41403
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.27.220.135:35331
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb | hidden | 127.27.220.136:39149
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb | hidden | 127.27.220.137:45251
tablet_copy_download_file_inject_latency_ms | 1500 | hidden | all 9 server(s) checked
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+----------------------+-------------------------
builtin_ntp_servers | 127.27.220.148:43555 | all 9 server(s) checked
time_source | builtin | all 9 server(s) checked
Version Summary
Version | Servers
-----------------+--------------------------
1.18.0-SNAPSHOT | all 10 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
rebalance_test_table_0 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_5 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_6 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_7 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_8 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_9 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 10
First Quartile | 10
Median | 10
Third Quartile | 10
Maximum | 10
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 9
Tables | 10
Tablets | 30
Replicas | 90
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 14:16:38.313614 28530 log_verifier.cc:126] Checking tablet 02adec8c6c704326afbedb70578df917
I20240502 14:16:38.358903 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 02adec8c6c704326afbedb70578df917
I20240502 14:16:38.359057 28530 log_verifier.cc:126] Checking tablet 03ebc1eccb85405a8f5ea7582d8822f9
I20240502 14:16:38.397378 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 03ebc1eccb85405a8f5ea7582d8822f9
I20240502 14:16:38.397475 28530 log_verifier.cc:126] Checking tablet 145a6a19b8474746a30ad51563419f1c
I20240502 14:16:38.471201 28530 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 145a6a19b8474746a30ad51563419f1c
I20240502 14:16:38.471315 28530 log_verifier.cc:126] Checking tablet 169109a7069249df905c760be1866c67
I20240502 14:16:38.507297 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 169109a7069249df905c760be1866c67
I20240502 14:16:38.507397 28530 log_verifier.cc:126] Checking tablet 24027cd85c0d4480bd1cd4639f4f6540
I20240502 14:16:38.543576 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 24027cd85c0d4480bd1cd4639f4f6540
I20240502 14:16:38.543689 28530 log_verifier.cc:126] Checking tablet 28f1bc6021e04a05916d3e49a8c98597
I20240502 14:16:38.581606 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 28f1bc6021e04a05916d3e49a8c98597
I20240502 14:16:38.581707 28530 log_verifier.cc:126] Checking tablet 3301dc9fca444af68c8a55dded735b38
I20240502 14:16:38.628062 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 3301dc9fca444af68c8a55dded735b38
I20240502 14:16:38.628206 28530 log_verifier.cc:126] Checking tablet 33e74e4a8ff147d69a8051f0a898fae6
I20240502 14:16:38.665305 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 33e74e4a8ff147d69a8051f0a898fae6
I20240502 14:16:38.665406 28530 log_verifier.cc:126] Checking tablet 3cc7001d1b9b425a8758f8e15914c289
I20240502 14:16:38.732955 28530 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 3cc7001d1b9b425a8758f8e15914c289
I20240502 14:16:38.733050 28530 log_verifier.cc:126] Checking tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b
I20240502 14:16:38.770962 28530 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 4aa8dd00e4bb46e2bd8ec2888a54d46b
I20240502 14:16:38.771073 28530 log_verifier.cc:126] Checking tablet 4f3030ae5dc2425586db21b6667e2deb
I20240502 14:16:38.812621 28530 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 4f3030ae5dc2425586db21b6667e2deb
I20240502 14:16:38.812765 28530 log_verifier.cc:126] Checking tablet 56aa6c21e5e947baaeaa6b42c89dfb8f
I20240502 14:16:38.854527 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 56aa6c21e5e947baaeaa6b42c89dfb8f
I20240502 14:16:38.854647 28530 log_verifier.cc:126] Checking tablet 609b87df71e64919a29a4fa093dfca23
I20240502 14:16:38.890271 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 609b87df71e64919a29a4fa093dfca23
I20240502 14:16:38.890422 28530 log_verifier.cc:126] Checking tablet 652af8d083784c078c5b45cca32ece4b
I20240502 14:16:38.929380 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 652af8d083784c078c5b45cca32ece4b
I20240502 14:16:38.929490 28530 log_verifier.cc:126] Checking tablet 6cdd66f946db45478a93459ba69702b2
I20240502 14:16:38.997241 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 6cdd66f946db45478a93459ba69702b2
I20240502 14:16:38.997334 28530 log_verifier.cc:126] Checking tablet 6f785b6443b741a483e16eb154439b27
I20240502 14:16:39.036091 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 6f785b6443b741a483e16eb154439b27
I20240502 14:16:39.036185 28530 log_verifier.cc:126] Checking tablet 7f7589fa40dd4af09cc87e6e793bcd8a
I20240502 14:16:39.074033 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 7f7589fa40dd4af09cc87e6e793bcd8a
I20240502 14:16:39.074138 28530 log_verifier.cc:126] Checking tablet 81be921c89194d3bb9c3d41be938ed74
I20240502 14:16:39.113214 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 81be921c89194d3bb9c3d41be938ed74
I20240502 14:16:39.113340 28530 log_verifier.cc:126] Checking tablet 88148a10290c4c48b87ab7e6aae5694c
I20240502 14:16:39.150774 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 88148a10290c4c48b87ab7e6aae5694c
I20240502 14:16:39.150904 28530 log_verifier.cc:126] Checking tablet 8dcd1e1699c8448692fff7c1ed36312a
I20240502 14:16:39.188385 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 8dcd1e1699c8448692fff7c1ed36312a
I20240502 14:16:39.188514 28530 log_verifier.cc:126] Checking tablet 9fd6780b78f849198bc91ed9ae2d08be
I20240502 14:16:39.258075 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 9fd6780b78f849198bc91ed9ae2d08be
I20240502 14:16:39.258180 28530 log_verifier.cc:126] Checking tablet a1a32a28425e442ca4846e423e457345
I20240502 14:16:39.295526 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet a1a32a28425e442ca4846e423e457345
I20240502 14:16:39.295646 28530 log_verifier.cc:126] Checking tablet a975b36b40b74545a122ff11a6e4bee1
I20240502 14:16:39.341720 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet a975b36b40b74545a122ff11a6e4bee1
I20240502 14:16:39.341850 28530 log_verifier.cc:126] Checking tablet aa0b526e216f46f0a2fb95fe0be02c46
I20240502 14:16:39.380218 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet aa0b526e216f46f0a2fb95fe0be02c46
I20240502 14:16:39.380348 28530 log_verifier.cc:126] Checking tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0
I20240502 14:16:39.417675 28530 log_verifier.cc:177] Verified matching terms for 6 ops in tablet b9b5ec9436264dd9bd1f4e51bb1a5fb0
I20240502 14:16:39.417785 28530 log_verifier.cc:126] Checking tablet cdaeb3ec3d344b4d9ac520f3d4c6d771
I20240502 14:16:39.457453 28530 log_verifier.cc:177] Verified matching terms for 10 ops in tablet cdaeb3ec3d344b4d9ac520f3d4c6d771
I20240502 14:16:39.457597 28530 log_verifier.cc:126] Checking tablet e46c5705a2aa4a3fbb410a284e5ea228
I20240502 14:16:39.528980 28530 log_verifier.cc:177] Verified matching terms for 7 ops in tablet e46c5705a2aa4a3fbb410a284e5ea228
I20240502 14:16:39.529088 28530 log_verifier.cc:126] Checking tablet e4f8998b30c34530a308f78e61138315
I20240502 14:16:39.566550 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet e4f8998b30c34530a308f78e61138315
I20240502 14:16:39.566673 28530 log_verifier.cc:126] Checking tablet e882402ebdb24b03a9c51d74dcfd3550
I20240502 14:16:39.607148 28530 log_verifier.cc:177] Verified matching terms for 10 ops in tablet e882402ebdb24b03a9c51d74dcfd3550
I20240502 14:16:39.607303 28530 log_verifier.cc:126] Checking tablet e9b6901a9a524fd2ab1adb9d730fcd18
I20240502 14:16:39.649567 28530 log_verifier.cc:177] Verified matching terms for 9 ops in tablet e9b6901a9a524fd2ab1adb9d730fcd18
I20240502 14:16:39.650367 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 2640
I20240502 14:16:39.659472 2762 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:39.912674 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 2640
W20240502 14:16:39.953497 4567 connection.cc:535] server connection from 127.27.220.129:59561 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:16:39.954427 5866 connection.cc:535] server connection from 127.27.220.129:50107 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:16:39.955011 2787 connection.cc:535] server connection from 127.27.220.129:45173 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:16:39.955132 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 2771
I20240502 14:16:39.965306 2892 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:16:40.101091 2917 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
W20240502 14:16:40.102730 2917 consensus_peers.cc:485] T b9b5ec9436264dd9bd1f4e51bb1a5fb0 P 8cee803d912546e1981521fd8f16bb4e -> Peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Couldn't send request to peer 6b3499bcb84b45a094f7166639704ed4. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:40.192169 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 2771
W20240502 14:16:40.232313 2919 connection.cc:535] client connection to 127.27.220.130:44843 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:16:40.232700 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 2901
W20240502 14:16:40.233721 2919 consensus_peers.cc:485] T 33e74e4a8ff147d69a8051f0a898fae6 P 8cee803d912546e1981521fd8f16bb4e -> Peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Couldn't send request to peer c6abc8535fe24d03bf71fb7092072fc4. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:40.238550 3022 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:16:40.297896 4568 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
W20240502 14:16:40.299381 4568 consensus_peers.cc:485] T 4f3030ae5dc2425586db21b6667e2deb P 6648fb2ed9f340ec9bd2d89ad88f2de4 -> Peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Couldn't send request to peer 6b3499bcb84b45a094f7166639704ed4. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:16:40.320374 4834 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111)
W20240502 14:16:40.321906 4834 consensus_peers.cc:485] T cdaeb3ec3d344b4d9ac520f3d4c6d771 P 21e5a60e02fd475bb47e5b3a2f294333 -> Peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Couldn't send request to peer c6abc8535fe24d03bf71fb7092072fc4. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:16:40.344925 4834 consensus_peers.cc:485] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 -> Peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Couldn't send request to peer c6abc8535fe24d03bf71fb7092072fc4. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:16:40.375054 3049 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111)
W20240502 14:16:40.376477 3049 consensus_peers.cc:485] T 88148a10290c4c48b87ab7e6aae5694c P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Couldn't send request to peer c6abc8535fe24d03bf71fb7092072fc4. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:40.471851 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 2901
W20240502 14:16:40.512522 3049 connection.cc:535] server connection from 127.27.220.131:52537 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 14:16:40.512748 3046 connection.cc:535] client connection to 127.27.220.131:36567 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:16:40.513124 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 3031
W20240502 14:16:40.514035 4831 consensus_peers.cc:485] T e882402ebdb24b03a9c51d74dcfd3550 P 21e5a60e02fd475bb47e5b3a2f294333 -> Peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): Couldn't send request to peer 8cee803d912546e1981521fd8f16bb4e. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:36567: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 14:16:40.514384 3046 consensus_peers.cc:485] T 8dcd1e1699c8448692fff7c1ed36312a P 472a7cc2e2d04cf0b36f7e43ed55f1d2 -> Peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): Couldn't send request to peer 8cee803d912546e1981521fd8f16bb4e. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:36567: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:40.519428 3152 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:16:40.672119 5863 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.131:36567: connect: Connection refused (error 111)
W20240502 14:16:40.673626 5863 consensus_peers.cc:485] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe -> Peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): Couldn't send request to peer 8cee803d912546e1981521fd8f16bb4e. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:36567: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:40.745786 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 3031
I20240502 14:16:40.781095 9140 raft_consensus.cc:482] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:40.781317 9140 raft_consensus.cc:504] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:40.782191 9140 leader_election.cc:288] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
W20240502 14:16:40.784128 4963 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
W20240502 14:16:40.786360 4963 leader_election.cc:334] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
I20240502 14:16:40.787012 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 4549
I20240502 14:16:40.791800 4633 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "652af8d083784c078c5b45cca32ece4b" candidate_uuid: "fa04f9da159541bfbbb6ac55db06e07b" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" is_pre_election: true
I20240502 14:16:40.792529 4963 leader_election.cc:302] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fa04f9da159541bfbbb6ac55db06e07b; no voters: 6648fb2ed9f340ec9bd2d89ad88f2de4, 6b3499bcb84b45a094f7166639704ed4
I20240502 14:16:40.792812 9140 raft_consensus.cc:2726] T 652af8d083784c078c5b45cca32ece4b P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:40.795043 4672 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:40.826614 9256 raft_consensus.cc:482] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:40.826911 9256 raft_consensus.cc:504] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:40.828491 9256 leader_election.cc:288] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
W20240502 14:16:40.829706 4700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
W20240502 14:16:40.831233 4702 leader_election.cc:334] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111)
W20240502 14:16:40.831478 4700 leader_election.cc:334] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
I20240502 14:16:40.831642 4700 leader_election.cc:302] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 763dcf9158e648b496b4faa1a0dc54af; no voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:16:40.831967 9256 raft_consensus.cc:2726] T 169109a7069249df905c760be1866c67 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:40.901181 9256 raft_consensus.cc:482] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:40.901453 9256 raft_consensus.cc:504] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } }
I20240502 14:16:40.902271 9256 leader_election.cc:288] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
W20240502 14:16:40.903808 4700 leader_election.cc:334] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
W20240502 14:16:40.903998 4699 leader_election.cc:334] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): Network error: Client connection negotiation failed: client connection to 127.27.220.131:36567: connect: Connection refused (error 111)
I20240502 14:16:40.904146 4699 leader_election.cc:302] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 763dcf9158e648b496b4faa1a0dc54af; no voters: 6b3499bcb84b45a094f7166639704ed4, 8cee803d912546e1981521fd8f16bb4e
I20240502 14:16:40.904410 9256 raft_consensus.cc:2726] T 03ebc1eccb85405a8f5ea7582d8822f9 P 763dcf9158e648b496b4faa1a0dc54af [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:40.954613 9247 raft_consensus.cc:482] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:40.954901 9247 raft_consensus.cc:504] T 652af8d083784c078c5b45cca32ece4b P 6648fb2ed9f340ec9bd2d89ad88f2de4 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:40.955224 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 4549
W20240502 14:16:40.978755 4964 consensus_peers.cc:485] T 3cc7001d1b9b425a8758f8e15914c289 P fa04f9da159541bfbbb6ac55db06e07b -> Peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): Couldn't send request to peer 472a7cc2e2d04cf0b36f7e43ed55f1d2. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.132:42265: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:40.988754 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 4684
I20240502 14:16:40.996397 4805 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:41.001118 9140 raft_consensus.cc:482] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:41.001308 9140 raft_consensus.cc:504] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:41.001955 9140 leader_election.cc:288] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331)
I20240502 14:16:41.002936 4896 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa0b526e216f46f0a2fb95fe0be02c46" candidate_uuid: "fa04f9da159541bfbbb6ac55db06e07b" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "21e5a60e02fd475bb47e5b3a2f294333" is_pre_election: true
W20240502 14:16:41.003312 4963 leader_election.cc:334] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
I20240502 14:16:41.003623 4963 leader_election.cc:302] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fa04f9da159541bfbbb6ac55db06e07b; no voters: 21e5a60e02fd475bb47e5b3a2f294333, 6b3499bcb84b45a094f7166639704ed4
I20240502 14:16:41.003893 9140 raft_consensus.cc:2726] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:41.075424 9253 raft_consensus.cc:482] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:41.075644 9253 raft_consensus.cc:504] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } }
I20240502 14:16:41.076557 9253 leader_election.cc:288] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787)
W20240502 14:16:41.078202 4834 leader_election.cc:334] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111)
W20240502 14:16:41.078418 4832 leader_election.cc:334] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
I20240502 14:16:41.078547 4832 leader_election.cc:302] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 21e5a60e02fd475bb47e5b3a2f294333; no voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:16:41.078806 9253 raft_consensus.cc:2726] T 6f785b6443b741a483e16eb154439b27 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20240502 14:16:41.112289 4833 consensus_peers.cc:485] T e46c5705a2aa4a3fbb410a284e5ea228 P 21e5a60e02fd475bb47e5b3a2f294333 -> Peer 472a7cc2e2d04cf0b36f7e43ed55f1d2 (127.27.220.132:42265): Couldn't send request to peer 472a7cc2e2d04cf0b36f7e43ed55f1d2. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.132:42265: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:41.135376 9213 raft_consensus.cc:482] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:41.135607 9213 raft_consensus.cc:504] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:41.136619 9213 leader_election.cc:288] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
I20240502 14:16:41.136895 9253 raft_consensus.cc:482] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:41.137077 9253 raft_consensus.cc:504] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
W20240502 14:16:41.138252 5864 leader_election.cc:334] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
W20240502 14:16:41.138485 5864 leader_election.cc:334] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Network error: Client connection negotiation failed: client connection to 127.27.220.133:41841: connect: Connection refused (error 111)
I20240502 14:16:41.138620 5864 leader_election.cc:302] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6fb8f2ad13f24e81935b7df7faa291fe; no voters: 6648fb2ed9f340ec9bd2d89ad88f2de4, 6b3499bcb84b45a094f7166639704ed4
I20240502 14:16:41.139047 9253 leader_election.cc:288] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149)
I20240502 14:16:41.139515 9263 raft_consensus.cc:2726] T 81be921c89194d3bb9c3d41be938ed74 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:41.139748 5028 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa0b526e216f46f0a2fb95fe0be02c46" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "fa04f9da159541bfbbb6ac55db06e07b" is_pre_election: true
I20240502 14:16:41.140000 5028 raft_consensus.cc:2454] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 1.
I20240502 14:16:41.140477 4832 leader_election.cc:302] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 21e5a60e02fd475bb47e5b3a2f294333, fa04f9da159541bfbbb6ac55db06e07b; no voters:
I20240502 14:16:41.140785 9253 raft_consensus.cc:2781] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Leader pre-election won for term 2
I20240502 14:16:41.140874 9253 raft_consensus.cc:482] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Starting leader election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:41.140951 9253 raft_consensus.cc:3037] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 1 FOLLOWER]: Advancing to term 2
W20240502 14:16:41.142247 4832 leader_election.cc:334] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
I20240502 14:16:41.143852 9253 raft_consensus.cc:504] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Starting leader election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:41.144472 9253 leader_election.cc:288] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: Requested vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), fa04f9da159541bfbbb6ac55db06e07b (127.27.220.136:39149)
I20240502 14:16:41.145113 5028 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa0b526e216f46f0a2fb95fe0be02c46" candidate_uuid: "21e5a60e02fd475bb47e5b3a2f294333" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "fa04f9da159541bfbbb6ac55db06e07b"
I20240502 14:16:41.145287 5028 raft_consensus.cc:3037] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Advancing to term 2
W20240502 14:16:41.145541 4832 leader_election.cc:334] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
I20240502 14:16:41.147926 5028 raft_consensus.cc:2454] T aa0b526e216f46f0a2fb95fe0be02c46 P fa04f9da159541bfbbb6ac55db06e07b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 21e5a60e02fd475bb47e5b3a2f294333 in term 2.
I20240502 14:16:41.148350 4832 leader_election.cc:302] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 21e5a60e02fd475bb47e5b3a2f294333, fa04f9da159541bfbbb6ac55db06e07b; no voters: 6b3499bcb84b45a094f7166639704ed4
I20240502 14:16:41.148627 9253 raft_consensus.cc:2781] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:41.148766 9253 raft_consensus.cc:686] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [term 2 LEADER]: Becoming Leader. State: Replica: 21e5a60e02fd475bb47e5b3a2f294333, State: Running, Role: LEADER
I20240502 14:16:41.149014 9253 consensus_queue.cc:237] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:41.151742 2575 catalog_manager.cc:5484] T aa0b526e216f46f0a2fb95fe0be02c46 P 21e5a60e02fd475bb47e5b3a2f294333 reported cstate change: term changed from 1 to 2, leader changed from 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129) to 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135). New cstate: current_term: 2 leader_uuid: "21e5a60e02fd475bb47e5b3a2f294333" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:16:41.159049 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 4684
I20240502 14:16:41.160887 9140 raft_consensus.cc:482] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 8cee803d912546e1981521fd8f16bb4e)
I20240502 14:16:41.161024 9140 raft_consensus.cc:504] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "8cee803d912546e1981521fd8f16bb4e" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 36567 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } }
I20240502 14:16:41.161690 9140 leader_election.cc:288] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567), 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
W20240502 14:16:41.163100 4962 leader_election.cc:334] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8cee803d912546e1981521fd8f16bb4e (127.27.220.131:36567): Network error: Client connection negotiation failed: client connection to 127.27.220.131:36567: connect: Connection refused (error 111)
W20240502 14:16:41.163280 4963 leader_election.cc:334] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Network error: Client connection negotiation failed: client connection to 127.27.220.133:41841: connect: Connection refused (error 111)
I20240502 14:16:41.163411 4963 leader_election.cc:302] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fa04f9da159541bfbbb6ac55db06e07b; no voters: 6648fb2ed9f340ec9bd2d89ad88f2de4, 8cee803d912546e1981521fd8f16bb4e
I20240502 14:16:41.163694 9140 raft_consensus.cc:2726] T 24027cd85c0d4480bd1cd4639f4f6540 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:41.186776 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 4814
I20240502 14:16:41.192332 4937 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:41.219192 9213 raft_consensus.cc:482] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c6abc8535fe24d03bf71fb7092072fc4)
I20240502 14:16:41.219415 9213 raft_consensus.cc:504] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:41.220116 9213 leader_election.cc:288] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
W20240502 14:16:41.221814 5866 leader_election.cc:334] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111)
W20240502 14:16:41.221973 5864 leader_election.cc:334] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Network error: Client connection negotiation failed: client connection to 127.27.220.133:41841: connect: Connection refused (error 111)
I20240502 14:16:41.222137 5864 leader_election.cc:302] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6fb8f2ad13f24e81935b7df7faa291fe; no voters: 6648fb2ed9f340ec9bd2d89ad88f2de4, c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:16:41.222391 9213 raft_consensus.cc:2726] T e4f8998b30c34530a308f78e61138315 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:41.287425 9140 raft_consensus.cc:482] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c6abc8535fe24d03bf71fb7092072fc4)
I20240502 14:16:41.287678 9140 raft_consensus.cc:504] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "fa04f9da159541bfbbb6ac55db06e07b" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 39149 } attrs { promote: false } }
I20240502 14:16:41.288506 9140 leader_election.cc:288] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843)
W20240502 14:16:41.289791 4963 leader_election.cc:334] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
W20240502 14:16:41.290091 4965 leader_election.cc:334] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111)
I20240502 14:16:41.290241 4965 leader_election.cc:302] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: fa04f9da159541bfbbb6ac55db06e07b; no voters: 6b3499bcb84b45a094f7166639704ed4, c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:16:41.290520 9140 raft_consensus.cc:2726] T 3301dc9fca444af68c8a55dded735b38 P fa04f9da159541bfbbb6ac55db06e07b [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:41.346730 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 4814
I20240502 14:16:41.366179 9213 raft_consensus.cc:482] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c6abc8535fe24d03bf71fb7092072fc4)
I20240502 14:16:41.366453 9213 raft_consensus.cc:504] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "763dcf9158e648b496b4faa1a0dc54af" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 41403 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:41.367291 9213 leader_election.cc:288] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403)
W20240502 14:16:41.368827 5863 leader_election.cc:334] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Network error: Client connection negotiation failed: client connection to 127.27.220.134:41403: connect: Connection refused (error 111)
W20240502 14:16:41.369124 5866 leader_election.cc:334] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111)
I20240502 14:16:41.369247 5866 leader_election.cc:302] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6fb8f2ad13f24e81935b7df7faa291fe; no voters: 763dcf9158e648b496b4faa1a0dc54af, c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:16:41.369616 9213 raft_consensus.cc:2726] T e9b6901a9a524fd2ab1adb9d730fcd18 P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:41.374378 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 4946
I20240502 14:16:41.374981 9213 raft_consensus.cc:482] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting pre-election (detected failure of leader c6abc8535fe24d03bf71fb7092072fc4)
I20240502 14:16:41.375146 9213 raft_consensus.cc:504] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c6abc8535fe24d03bf71fb7092072fc4" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 44843 } } peers { permanent_uuid: "21e5a60e02fd475bb47e5b3a2f294333" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 35331 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:41.376011 9213 leader_election.cc:288] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843), 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331)
W20240502 14:16:41.377179 5866 leader_election.cc:334] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer c6abc8535fe24d03bf71fb7092072fc4 (127.27.220.130:44843): Network error: Client connection negotiation failed: client connection to 127.27.220.130:44843: connect: Connection refused (error 111)
W20240502 14:16:41.377393 5864 leader_election.cc:334] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 21e5a60e02fd475bb47e5b3a2f294333 (127.27.220.135:35331): Network error: Client connection negotiation failed: client connection to 127.27.220.135:35331: connect: Connection refused (error 111)
I20240502 14:16:41.377571 5864 leader_election.cc:302] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6fb8f2ad13f24e81935b7df7faa291fe; no voters: 21e5a60e02fd475bb47e5b3a2f294333, c6abc8535fe24d03bf71fb7092072fc4
I20240502 14:16:41.377830 9213 raft_consensus.cc:2726] T 56aa6c21e5e947baaeaa6b42c89dfb8f P 6fb8f2ad13f24e81935b7df7faa291fe [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 14:16:41.381062 5069 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 14:16:41.391055 5863 consensus_peers.cc:485] T 4aa8dd00e4bb46e2bd8ec2888a54d46b P 6fb8f2ad13f24e81935b7df7faa291fe -> Peer 763dcf9158e648b496b4faa1a0dc54af (127.27.220.134:41403): Couldn't send request to peer 763dcf9158e648b496b4faa1a0dc54af. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.134:41403: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:16:41.413075 9213 raft_consensus.cc:482] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 6b3499bcb84b45a094f7166639704ed4)
I20240502 14:16:41.413273 9213 raft_consensus.cc:504] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "6b3499bcb84b45a094f7166639704ed4" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 44787 } } peers { permanent_uuid: "6648fb2ed9f340ec9bd2d89ad88f2de4" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 41841 } attrs { promote: false } } peers { permanent_uuid: "6fb8f2ad13f24e81935b7df7faa291fe" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 45251 } attrs { promote: false } }
I20240502 14:16:41.413832 9213 leader_election.cc:288] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787), 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841)
W20240502 14:16:41.415048 5864 leader_election.cc:334] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6b3499bcb84b45a094f7166639704ed4 (127.27.220.129:44787): Network error: Client connection negotiation failed: client connection to 127.27.220.129:44787: connect: Connection refused (error 111)
W20240502 14:16:41.415217 5864 leader_election.cc:334] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer 6648fb2ed9f340ec9bd2d89ad88f2de4 (127.27.220.133:41841): Network error: Client connection negotiation failed: client connection to 127.27.220.133:41841: connect: Connection refused (error 111)
I20240502 14:16:41.415297 5864 leader_election.cc:302] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 6fb8f2ad13f24e81935b7df7faa291fe; no voters: 6648fb2ed9f340ec9bd2d89ad88f2de4, 6b3499bcb84b45a094f7166639704ed4
I20240502 14:16:41.415585 9213 raft_consensus.cc:2726] T 145a6a19b8474746a30ad51563419f1c P 6fb8f2ad13f24e81935b7df7faa291fe [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
I20240502 14:16:41.528815 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 4946
I20240502 14:16:41.554838 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 5744
I20240502 14:16:41.560837 5971 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:41.685824 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 5744
I20240502 14:16:41.709441 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 2550
I20240502 14:16:41.710577 2611 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:41.916930 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 2550
2024-05-02T14:16:41Z chronyd exiting
[ OK ] TserverAddedDuringRebalancingTest.TserverStarts/1 (53841 ms)
[----------] 1 test from TserverAddedDuringRebalancingTest (53841 ms total)
[----------] 1 test from RebalancerAndSingleReplicaTablets
[ RUN ] RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove/5
I20240502 14:16:42.088729 28530 test_util.cc:268] Using random seed: 1083343094
I20240502 14:16:42.090667 28530 ts_itest-base.cc:112] Starting cluster with:
I20240502 14:16:42.090739 28530 ts_itest-base.cc:113] --------------
I20240502 14:16:42.090814 28530 ts_itest-base.cc:114] 4 tablet servers
I20240502 14:16:42.090893 28530 ts_itest-base.cc:115] 1 replicas per TS
I20240502 14:16:42.090945 28530 ts_itest-base.cc:116] --------------
2024-05-02T14:16:42Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T14:16:42Z Disabled control of system clock
I20240502 14:16:42.140573 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:42681
--webserver_interface=127.27.220.190
--webserver_port=0
--builtin_ntp_servers=127.27.220.148:46499
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.27.220.190:42681
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false with env {}
W20240502 14:16:42.472462 9285 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:42.472844 9285 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:42.473001 9285 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:42.480829 9285 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:42.480996 9285 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 14:16:42.481101 9285 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:42.481194 9285 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 14:16:42.481274 9285 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 14:16:42.491269 9285 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:46499
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.27.220.190:42681
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:42681
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.27.220.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:42.492743 9285 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:42.494616 9285 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:42.505288 9290 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:42.505924 9293 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:42.506460 9291 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:42.507181 9285 server_base.cc:1034] running on GCE node
I20240502 14:16:42.507824 9285 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:42.509063 9285 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:42.510259 9285 hybrid_clock.cc:648] HybridClock initialized: now 1714659402510206 us; error 46 us; skew 500 ppm
I20240502 14:16:42.510589 9285 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:42.512732 9285 webserver.cc:460] Webserver started at http://127.27.220.190:45743/ using document root <none> and password file <none>
I20240502 14:16:42.513442 9285 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:42.513577 9285 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:42.513898 9285 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:42.516268 9285 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e"
format_stamp: "Formatted at 2024-05-02 14:16:42 on dist-test-slave-j72s"
I20240502 14:16:42.516980 9285 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e"
format_stamp: "Formatted at 2024-05-02 14:16:42 on dist-test-slave-j72s"
I20240502 14:16:42.521646 9285 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.007s sys 0.000s
I20240502 14:16:42.524641 9299 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:42.525823 9285 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 14:16:42.525979 9285 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e"
format_stamp: "Formatted at 2024-05-02 14:16:42 on dist-test-slave-j72s"
I20240502 14:16:42.526154 9285 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:42.540822 9285 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:42.541610 9285 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:42.541878 9285 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:42.558193 9285 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.190:42681
I20240502 14:16:42.558211 9350 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.190:42681 every 8 connection(s)
I20240502 14:16:42.559360 9285 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 14:16:42.562825 9351 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:42.565928 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 9285
I20240502 14:16:42.566164 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 14:16:42.575415 9351 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e: Bootstrap starting.
I20240502 14:16:42.580094 9351 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:42.581481 9351 log.cc:824] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:42.584522 9351 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e: No bootstrap required, opened a new log
I20240502 14:16:42.594741 9351 raft_consensus.cc:348] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42681 } }
I20240502 14:16:42.595216 9351 raft_consensus.cc:374] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:42.595341 9351 raft_consensus.cc:729] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 9a5481ee95bc467b88f3c0b3ca1cbf0e, State: Initialized, Role: FOLLOWER
I20240502 14:16:42.596099 9351 consensus_queue.cc:260] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42681 } }
I20240502 14:16:42.596400 9351 raft_consensus.cc:388] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:42.596531 9351 raft_consensus.cc:482] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:42.596697 9351 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:42.600061 9351 raft_consensus.cc:504] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42681 } }
I20240502 14:16:42.600709 9351 leader_election.cc:302] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 9a5481ee95bc467b88f3c0b3ca1cbf0e; no voters:
I20240502 14:16:42.601413 9351 leader_election.cc:288] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:42.601557 9356 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:42.602164 9356 raft_consensus.cc:686] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [term 1 LEADER]: Becoming Leader. State: Replica: 9a5481ee95bc467b88f3c0b3ca1cbf0e, State: Running, Role: LEADER
I20240502 14:16:42.602767 9356 consensus_queue.cc:237] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42681 } }
I20240502 14:16:42.603716 9351 sys_catalog.cc:564] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [sys.catalog]: configured and running, proceeding with master startup.
I20240502 14:16:42.605181 9358 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [sys.catalog]: SysCatalogTable state changed. Reason: New leader 9a5481ee95bc467b88f3c0b3ca1cbf0e. Latest consensus state: current_term: 1 leader_uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42681 } } }
I20240502 14:16:42.605571 9358 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [sys.catalog]: This master's current role is: LEADER
I20240502 14:16:42.606166 9357 sys_catalog.cc:455] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "9a5481ee95bc467b88f3c0b3ca1cbf0e" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 42681 } } }
I20240502 14:16:42.606544 9357 sys_catalog.cc:458] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e [sys.catalog]: This master's current role is: LEADER
I20240502 14:16:42.607817 9362 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 14:16:42.613346 9362 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 14:16:42.628186 9374 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
I20240502 14:16:42.631088 9362 catalog_manager.cc:1347] Generated new cluster ID: 6d4b3ce06f5a4b76a7059ddeab76464b
I20240502 14:16:42.631203 9362 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 14:16:42.645504 9362 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 14:16:42.646831 9362 catalog_manager.cc:1504] Loading token signing keys...
I20240502 14:16:42.666055 9362 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 9a5481ee95bc467b88f3c0b3ca1cbf0e: Generated new TSK 0
I20240502 14:16:42.667114 9362 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 14:16:42.674705 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.129:0
--local_ip_for_outbound_sockets=127.27.220.129
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42681
--builtin_ntp_servers=127.27.220.148:46499
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:16:43.010788 9376 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:43.011052 9376 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:43.011154 9376 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:43.018302 9376 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:43.018435 9376 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:43.018541 9376 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.129
I20240502 14:16:43.026870 9376 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:46499
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42681
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.129
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:43.028008 9376 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:43.029807 9376 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:43.041551 9381 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:43.042078 9382 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:43.042495 9384 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:43.043177 9376 server_base.cc:1034] running on GCE node
I20240502 14:16:43.043740 9376 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:43.044566 9376 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:43.045754 9376 hybrid_clock.cc:648] HybridClock initialized: now 1714659403045698 us; error 44 us; skew 500 ppm
I20240502 14:16:43.046037 9376 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:43.048195 9376 webserver.cc:460] Webserver started at http://127.27.220.129:39727/ using document root <none> and password file <none>
I20240502 14:16:43.048882 9376 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:43.048982 9376 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:43.049233 9376 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:43.051753 9376 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "12fda910bc6142fb8f2d9aceffa40631"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.052498 9376 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "12fda910bc6142fb8f2d9aceffa40631"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.057447 9376 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.004s
I20240502 14:16:43.060636 9390 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:43.061942 9376 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.003s
I20240502 14:16:43.062085 9376 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "12fda910bc6142fb8f2d9aceffa40631"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.062258 9376 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:43.082140 9376 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:43.082860 9376 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:43.083073 9376 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:43.083884 9376 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:43.085295 9376 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:43.085384 9376 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:43.085464 9376 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:43.085538 9376 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:43.122573 9376 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.129:37173
I20240502 14:16:43.122627 9502 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.129:37173 every 8 connection(s)
I20240502 14:16:43.123795 9376 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 14:16:43.124945 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 9376
I20240502 14:16:43.125144 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 14:16:43.128726 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.130:0
--local_ip_for_outbound_sockets=127.27.220.130
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42681
--builtin_ntp_servers=127.27.220.148:46499
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:16:43.144348 9503 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42681
I20240502 14:16:43.144634 9503 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:43.145380 9503 heartbeater.cc:511] Master 127.27.220.190:42681 requested a full tablet report, sending...
I20240502 14:16:43.147485 9316 ts_manager.cc:194] Registered new tserver with Master: 12fda910bc6142fb8f2d9aceffa40631 (127.27.220.129:37173)
I20240502 14:16:43.149623 9316 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.129:47163
W20240502 14:16:43.461019 9507 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:43.461318 9507 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:43.461416 9507 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:43.469184 9507 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:43.469311 9507 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:43.469432 9507 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.130
I20240502 14:16:43.478619 9507 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:46499
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42681
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.130
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:43.479841 9507 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:43.481505 9507 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:43.492849 9512 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:43.493346 9513 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:43.493922 9515 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:43.494397 9507 server_base.cc:1034] running on GCE node
I20240502 14:16:43.494890 9507 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:43.495759 9507 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:43.496982 9507 hybrid_clock.cc:648] HybridClock initialized: now 1714659403496929 us; error 36 us; skew 500 ppm
I20240502 14:16:43.497295 9507 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:43.499460 9507 webserver.cc:460] Webserver started at http://127.27.220.130:33013/ using document root <none> and password file <none>
I20240502 14:16:43.500152 9507 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:43.500275 9507 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:43.500572 9507 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:43.503116 9507 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "46d58e4957b949d8a1491d833eca1b6c"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.503764 9507 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "46d58e4957b949d8a1491d833eca1b6c"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.508352 9507 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.001s
I20240502 14:16:43.511786 9521 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:43.513015 9507 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:43.513155 9507 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "46d58e4957b949d8a1491d833eca1b6c"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.513332 9507 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:43.529422 9507 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:43.530191 9507 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:43.530421 9507 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:43.531281 9507 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:43.532614 9507 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:43.532699 9507 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:43.532776 9507 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:43.532829 9507 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:43.564052 9507 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.130:38941
I20240502 14:16:43.564090 9633 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.130:38941 every 8 connection(s)
I20240502 14:16:43.565249 9507 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 14:16:43.571780 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 9507
I20240502 14:16:43.571969 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 14:16:43.575582 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:0
--local_ip_for_outbound_sockets=127.27.220.131
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42681
--builtin_ntp_servers=127.27.220.148:46499
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:16:43.578670 9634 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42681
I20240502 14:16:43.578967 9634 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:43.579914 9634 heartbeater.cc:511] Master 127.27.220.190:42681 requested a full tablet report, sending...
I20240502 14:16:43.581431 9316 ts_manager.cc:194] Registered new tserver with Master: 46d58e4957b949d8a1491d833eca1b6c (127.27.220.130:38941)
I20240502 14:16:43.582190 9316 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.130:38175
W20240502 14:16:43.891923 9637 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:43.892223 9637 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:43.892351 9637 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:43.899217 9637 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:43.899360 9637 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:43.899508 9637 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:16:43.907907 9637 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:46499
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42681
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:43.909200 9637 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:43.911000 9637 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:43.922641 9642 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:43.923027 9645 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:43.923555 9643 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:43.924223 9637 server_base.cc:1034] running on GCE node
I20240502 14:16:43.924752 9637 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:43.925647 9637 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:43.926854 9637 hybrid_clock.cc:648] HybridClock initialized: now 1714659403926795 us; error 48 us; skew 500 ppm
I20240502 14:16:43.927167 9637 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:43.929250 9637 webserver.cc:460] Webserver started at http://127.27.220.131:35879/ using document root <none> and password file <none>
I20240502 14:16:43.930054 9637 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:43.930167 9637 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:43.930497 9637 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:43.933084 9637 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "5802fae32c644b9ba3baceb911d98967"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.933859 9637 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "5802fae32c644b9ba3baceb911d98967"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.938973 9637 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.002s
I20240502 14:16:43.942229 9651 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:43.943493 9637 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
I20240502 14:16:43.943746 9637 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "5802fae32c644b9ba3baceb911d98967"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:43.943995 9637 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:43.968605 9637 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:43.969486 9637 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:43.969849 9637 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:43.970847 9637 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:43.972565 9637 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:43.972803 9637 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:43.972945 9637 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:43.973029 9637 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:44.008985 9637 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:38377
I20240502 14:16:44.009061 9763 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:38377 every 8 connection(s)
I20240502 14:16:44.010569 9637 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:16:44.016534 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 9637
I20240502 14:16:44.016683 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 14:16:44.020052 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.132:0
--local_ip_for_outbound_sockets=127.27.220.132
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42681
--builtin_ntp_servers=127.27.220.148:46499
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:16:44.024101 9764 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42681
I20240502 14:16:44.024403 9764 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:44.025323 9764 heartbeater.cc:511] Master 127.27.220.190:42681 requested a full tablet report, sending...
I20240502 14:16:44.026829 9316 ts_manager.cc:194] Registered new tserver with Master: 5802fae32c644b9ba3baceb911d98967 (127.27.220.131:38377)
I20240502 14:16:44.027558 9316 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:39485
I20240502 14:16:44.152426 9503 heartbeater.cc:503] Master 127.27.220.190:42681 was elected leader, sending a full tablet report...
W20240502 14:16:44.354485 9767 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:44.354786 9767 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:44.355077 9767 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:44.362250 9767 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:44.362391 9767 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:44.362540 9767 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.132
I20240502 14:16:44.371152 9767 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:46499
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:42681
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.132
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:44.372391 9767 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:44.374064 9767 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:44.385265 9772 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:44.385892 9775 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:44.386379 9773 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:44.387073 9767 server_base.cc:1034] running on GCE node
I20240502 14:16:44.387589 9767 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:44.388484 9767 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:44.389691 9767 hybrid_clock.cc:648] HybridClock initialized: now 1714659404389655 us; error 32 us; skew 500 ppm
I20240502 14:16:44.389990 9767 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:44.392258 9767 webserver.cc:460] Webserver started at http://127.27.220.132:45345/ using document root <none> and password file <none>
I20240502 14:16:44.393047 9767 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:44.393177 9767 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:44.393481 9767 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:44.395969 9767 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "782bc0b09bad4847afe30ebacd237298"
format_stamp: "Formatted at 2024-05-02 14:16:44 on dist-test-slave-j72s"
I20240502 14:16:44.396684 9767 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "782bc0b09bad4847afe30ebacd237298"
format_stamp: "Formatted at 2024-05-02 14:16:44 on dist-test-slave-j72s"
I20240502 14:16:44.401439 9767 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.007s sys 0.000s
I20240502 14:16:44.404764 9781 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:44.405995 9767 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.003s
I20240502 14:16:44.406139 9767 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "782bc0b09bad4847afe30ebacd237298"
format_stamp: "Formatted at 2024-05-02 14:16:44 on dist-test-slave-j72s"
I20240502 14:16:44.406316 9767 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:44.432345 9767 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:44.433138 9767 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:44.433432 9767 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:44.434366 9767 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:44.435776 9767 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:44.435869 9767 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:44.435988 9767 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:44.436060 9767 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:44.468286 9767 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.132:38413
I20240502 14:16:44.468358 9893 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.132:38413 every 8 connection(s)
I20240502 14:16:44.469599 9767 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 14:16:44.471380 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 9767
I20240502 14:16:44.471583 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 14:16:44.482353 9894 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42681
I20240502 14:16:44.482591 9894 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:44.483284 9894 heartbeater.cc:511] Master 127.27.220.190:42681 requested a full tablet report, sending...
I20240502 14:16:44.484463 9316 ts_manager.cc:194] Registered new tserver with Master: 782bc0b09bad4847afe30ebacd237298 (127.27.220.132:38413)
I20240502 14:16:44.485225 9316 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.132:43631
I20240502 14:16:44.487015 28530 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 14:16:44.501391 9316 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36978:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 14:16:44.529237 9699 tablet_service.cc:1467] Processing CreateTablet for tablet 73f5d7eb8a334aa78ca245efdc1e092f (DEFAULT_TABLE table=TestTable [id=e1cdf798f1164c219e10c05c4c6abff9]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:44.530478 9699 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 73f5d7eb8a334aa78ca245efdc1e092f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:44.539723 9908 tablet_bootstrap.cc:492] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Bootstrap starting.
I20240502 14:16:44.542573 9908 tablet_bootstrap.cc:654] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:44.543586 9908 log.cc:824] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:44.545755 9908 tablet_bootstrap.cc:492] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: No bootstrap required, opened a new log
I20240502 14:16:44.545994 9908 ts_tablet_manager.cc:1367] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Time spent bootstrapping tablet: real 0.006s user 0.006s sys 0.000s
I20240502 14:16:44.551988 9908 raft_consensus.cc:348] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } }
I20240502 14:16:44.552320 9908 raft_consensus.cc:374] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:44.552413 9908 raft_consensus.cc:729] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5802fae32c644b9ba3baceb911d98967, State: Initialized, Role: FOLLOWER
I20240502 14:16:44.552958 9908 consensus_queue.cc:260] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } }
I20240502 14:16:44.553156 9908 raft_consensus.cc:388] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:44.553275 9908 raft_consensus.cc:482] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:44.553431 9908 raft_consensus.cc:3037] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:44.555299 9908 raft_consensus.cc:504] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } }
I20240502 14:16:44.555745 9908 leader_election.cc:302] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 5802fae32c644b9ba3baceb911d98967; no voters:
I20240502 14:16:44.556286 9908 leader_election.cc:288] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:44.556425 9910 raft_consensus.cc:2781] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:44.556994 9910 raft_consensus.cc:686] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 1 LEADER]: Becoming Leader. State: Replica: 5802fae32c644b9ba3baceb911d98967, State: Running, Role: LEADER
I20240502 14:16:44.557391 9764 heartbeater.cc:503] Master 127.27.220.190:42681 was elected leader, sending a full tablet report...
I20240502 14:16:44.557926 9908 ts_tablet_manager.cc:1398] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Time spent starting tablet: real 0.012s user 0.012s sys 0.000s
I20240502 14:16:44.558112 9910 consensus_queue.cc:237] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } }
I20240502 14:16:44.563588 9316 catalog_manager.cc:5484] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 reported cstate change: term changed from 0 to 1, leader changed from <none> to 5802fae32c644b9ba3baceb911d98967 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "5802fae32c644b9ba3baceb911d98967" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:44.574795 28530 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 14:16:44.576725 28530 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 12fda910bc6142fb8f2d9aceffa40631 to finish bootstrapping
I20240502 14:16:44.584748 9634 heartbeater.cc:503] Master 127.27.220.190:42681 was elected leader, sending a full tablet report...
I20240502 14:16:44.585362 28530 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 46d58e4957b949d8a1491d833eca1b6c to finish bootstrapping
I20240502 14:16:44.593132 28530 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 5802fae32c644b9ba3baceb911d98967 to finish bootstrapping
I20240502 14:16:44.599237 28530 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 782bc0b09bad4847afe30ebacd237298 to finish bootstrapping
I20240502 14:16:44.607282 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 9637
I20240502 14:16:44.608080 9759 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:44.690249 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 9637
I20240502 14:16:44.707531 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 9767
I20240502 14:16:44.708576 9889 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:44.786418 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 9767
I20240502 14:16:48.554495 9315 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36978:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:48.590073 9569 tablet_service.cc:1467] Processing CreateTablet for tablet 6145e4ff8fd34366bd1a2eddc988eb0e (DEFAULT_TABLE table=rebalance_test_table_0 [id=a2145fc5ee8642a482360ba4f0981231]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:48.590444 9437 tablet_service.cc:1467] Processing CreateTablet for tablet c245d4a31c5440c886c443d91d20c1ff (DEFAULT_TABLE table=rebalance_test_table_0 [id=a2145fc5ee8642a482360ba4f0981231]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:48.590416 9438 tablet_service.cc:1467] Processing CreateTablet for tablet 6cdbded6369f45e397c214e368c5aed6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=a2145fc5ee8642a482360ba4f0981231]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:48.591753 9569 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6145e4ff8fd34366bd1a2eddc988eb0e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:48.592206 9438 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6cdbded6369f45e397c214e368c5aed6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:48.592828 9437 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c245d4a31c5440c886c443d91d20c1ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:48.603439 9929 tablet_bootstrap.cc:492] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c: Bootstrap starting.
I20240502 14:16:48.603971 9930 tablet_bootstrap.cc:492] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631: Bootstrap starting.
I20240502 14:16:48.607102 9929 tablet_bootstrap.cc:654] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:48.607328 9930 tablet_bootstrap.cc:654] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:48.608486 9929 log.cc:824] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:48.608486 9930 log.cc:824] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:48.611434 9929 tablet_bootstrap.cc:492] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c: No bootstrap required, opened a new log
I20240502 14:16:48.611719 9929 ts_tablet_manager.cc:1367] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c: Time spent bootstrapping tablet: real 0.009s user 0.006s sys 0.000s
I20240502 14:16:48.611876 9930 tablet_bootstrap.cc:492] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631: No bootstrap required, opened a new log
I20240502 14:16:48.612169 9930 ts_tablet_manager.cc:1367] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
I20240502 14:16:48.620815 9929 raft_consensus.cc:348] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:48.620893 9930 raft_consensus.cc:348] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:48.621254 9929 raft_consensus.cc:374] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:48.621254 9930 raft_consensus.cc:374] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:48.621361 9930 raft_consensus.cc:729] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Initialized, Role: FOLLOWER
I20240502 14:16:48.621425 9929 raft_consensus.cc:729] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Initialized, Role: FOLLOWER
I20240502 14:16:48.622125 9929 consensus_queue.cc:260] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:48.622125 9930 consensus_queue.cc:260] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:48.622390 9930 raft_consensus.cc:388] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:48.622395 9929 raft_consensus.cc:388] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:48.622545 9930 raft_consensus.cc:482] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:48.622539 9929 raft_consensus.cc:482] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:48.622727 9929 raft_consensus.cc:3037] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:48.622745 9930 raft_consensus.cc:3037] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:48.625800 9929 raft_consensus.cc:504] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:48.625792 9930 raft_consensus.cc:504] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:48.626370 9929 leader_election.cc:302] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 46d58e4957b949d8a1491d833eca1b6c; no voters:
I20240502 14:16:48.626370 9930 leader_election.cc:302] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 12fda910bc6142fb8f2d9aceffa40631; no voters:
I20240502 14:16:48.627089 9930 leader_election.cc:288] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:48.627183 9933 raft_consensus.cc:2781] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:48.627573 9929 leader_election.cc:288] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:48.627705 9934 raft_consensus.cc:2781] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:48.631812 9929 ts_tablet_manager.cc:1398] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c: Time spent starting tablet: real 0.020s user 0.006s sys 0.014s
I20240502 14:16:48.633062 9933 raft_consensus.cc:686] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 LEADER]: Becoming Leader. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Running, Role: LEADER
I20240502 14:16:48.633675 9930 ts_tablet_manager.cc:1398] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631: Time spent starting tablet: real 0.021s user 0.019s sys 0.004s
I20240502 14:16:48.633940 9934 raft_consensus.cc:686] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [term 1 LEADER]: Becoming Leader. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Running, Role: LEADER
I20240502 14:16:48.633852 9933 consensus_queue.cc:237] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:48.634186 9930 tablet_bootstrap.cc:492] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631: Bootstrap starting.
I20240502 14:16:48.634459 9934 consensus_queue.cc:237] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:48.635699 9930 tablet_bootstrap.cc:654] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:48.639308 9316 catalog_manager.cc:5484] T 6cdbded6369f45e397c214e368c5aed6 P 12fda910bc6142fb8f2d9aceffa40631 reported cstate change: term changed from 0 to 1, leader changed from <none> to 12fda910bc6142fb8f2d9aceffa40631 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "12fda910bc6142fb8f2d9aceffa40631" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:48.639308 9315 catalog_manager.cc:5484] T 6145e4ff8fd34366bd1a2eddc988eb0e P 46d58e4957b949d8a1491d833eca1b6c reported cstate change: term changed from 0 to 1, leader changed from <none> to 46d58e4957b949d8a1491d833eca1b6c (127.27.220.130). New cstate: current_term: 1 leader_uuid: "46d58e4957b949d8a1491d833eca1b6c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:48.644412 9930 tablet_bootstrap.cc:492] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631: No bootstrap required, opened a new log
I20240502 14:16:48.644615 9930 ts_tablet_manager.cc:1367] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.008s
I20240502 14:16:48.645610 9930 raft_consensus.cc:348] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:48.645874 9930 raft_consensus.cc:374] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:48.646051 9930 raft_consensus.cc:729] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Initialized, Role: FOLLOWER
I20240502 14:16:48.646394 9930 consensus_queue.cc:260] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:48.646577 9930 raft_consensus.cc:388] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:48.646659 9930 raft_consensus.cc:482] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:48.646762 9930 raft_consensus.cc:3037] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:48.649107 9930 raft_consensus.cc:504] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:48.649339 9930 leader_election.cc:302] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 12fda910bc6142fb8f2d9aceffa40631; no voters:
I20240502 14:16:48.649526 9930 leader_election.cc:288] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:48.649602 9933 raft_consensus.cc:2781] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:48.649952 9930 ts_tablet_manager.cc:1398] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 14:16:48.650028 9933 raft_consensus.cc:686] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [term 1 LEADER]: Becoming Leader. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Running, Role: LEADER
I20240502 14:16:48.650260 9933 consensus_queue.cc:237] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:48.652269 9316 catalog_manager.cc:5484] T c245d4a31c5440c886c443d91d20c1ff P 12fda910bc6142fb8f2d9aceffa40631 reported cstate change: term changed from 0 to 1, leader changed from <none> to 12fda910bc6142fb8f2d9aceffa40631 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "12fda910bc6142fb8f2d9aceffa40631" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:49.160733 9316 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36978:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:49.169175 9568 tablet_service.cc:1467] Processing CreateTablet for tablet 451589dd12df44da9e64bf048329f841 (DEFAULT_TABLE table=rebalance_test_table_1 [id=d624c237e0aa4862868f4e0d395c292e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:49.169348 9569 tablet_service.cc:1467] Processing CreateTablet for tablet fdc2fee6e20d4e558f8f8366da152ab9 (DEFAULT_TABLE table=rebalance_test_table_1 [id=d624c237e0aa4862868f4e0d395c292e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:49.169315 9438 tablet_service.cc:1467] Processing CreateTablet for tablet 542b5a718dc843bc8e9b623ba560d5a6 (DEFAULT_TABLE table=rebalance_test_table_1 [id=d624c237e0aa4862868f4e0d395c292e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:49.169672 9568 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 451589dd12df44da9e64bf048329f841. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:49.169799 9438 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 542b5a718dc843bc8e9b623ba560d5a6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:49.169937 9569 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fdc2fee6e20d4e558f8f8366da152ab9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:49.175546 9968 tablet_bootstrap.cc:492] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c: Bootstrap starting.
I20240502 14:16:49.176648 9969 tablet_bootstrap.cc:492] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631: Bootstrap starting.
I20240502 14:16:49.177371 9968 tablet_bootstrap.cc:654] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:49.178376 9969 tablet_bootstrap.cc:654] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:49.179502 9968 tablet_bootstrap.cc:492] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c: No bootstrap required, opened a new log
I20240502 14:16:49.179639 9968 ts_tablet_manager.cc:1367] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:16:49.180142 9969 tablet_bootstrap.cc:492] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631: No bootstrap required, opened a new log
I20240502 14:16:49.180285 9969 ts_tablet_manager.cc:1367] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:16:49.180642 9968 raft_consensus.cc:348] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.180822 9968 raft_consensus.cc:374] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:49.180924 9968 raft_consensus.cc:729] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Initialized, Role: FOLLOWER
I20240502 14:16:49.181001 9969 raft_consensus.cc:348] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.181174 9969 raft_consensus.cc:374] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:49.181176 9968 consensus_queue.cc:260] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.181285 9969 raft_consensus.cc:729] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Initialized, Role: FOLLOWER
I20240502 14:16:49.181398 9968 raft_consensus.cc:388] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:49.181529 9968 raft_consensus.cc:482] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:49.181653 9968 raft_consensus.cc:3037] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:49.181753 9969 consensus_queue.cc:260] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.181967 9969 raft_consensus.cc:388] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:49.182053 9969 raft_consensus.cc:482] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:49.182171 9969 raft_consensus.cc:3037] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:49.184224 9968 raft_consensus.cc:504] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.184475 9968 leader_election.cc:302] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 46d58e4957b949d8a1491d833eca1b6c; no voters:
I20240502 14:16:49.184700 9968 leader_election.cc:288] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:49.184692 9969 raft_consensus.cc:504] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.184791 9938 raft_consensus.cc:2781] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:49.184994 9969 leader_election.cc:302] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 12fda910bc6142fb8f2d9aceffa40631; no voters:
I20240502 14:16:49.185289 9938 raft_consensus.cc:686] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [term 1 LEADER]: Becoming Leader. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Running, Role: LEADER
I20240502 14:16:49.185633 9938 consensus_queue.cc:237] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.187891 9316 catalog_manager.cc:5484] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c reported cstate change: term changed from 0 to 1, leader changed from <none> to 46d58e4957b949d8a1491d833eca1b6c (127.27.220.130). New cstate: current_term: 1 leader_uuid: "46d58e4957b949d8a1491d833eca1b6c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:49.189293 9968 ts_tablet_manager.cc:1398] T 451589dd12df44da9e64bf048329f841 P 46d58e4957b949d8a1491d833eca1b6c: Time spent starting tablet: real 0.010s user 0.006s sys 0.005s
I20240502 14:16:49.189647 9968 tablet_bootstrap.cc:492] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c: Bootstrap starting.
I20240502 14:16:49.189734 9969 leader_election.cc:288] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:49.190016 9974 raft_consensus.cc:2781] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:49.191541 9968 tablet_bootstrap.cc:654] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:49.196492 9969 ts_tablet_manager.cc:1398] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631: Time spent starting tablet: real 0.016s user 0.009s sys 0.004s
I20240502 14:16:49.196862 9974 raft_consensus.cc:686] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 LEADER]: Becoming Leader. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Running, Role: LEADER
I20240502 14:16:49.197190 9968 tablet_bootstrap.cc:492] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c: No bootstrap required, opened a new log
I20240502 14:16:49.197178 9974 consensus_queue.cc:237] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.197365 9968 ts_tablet_manager.cc:1367] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.004s
I20240502 14:16:49.198173 9968 raft_consensus.cc:348] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.198362 9968 raft_consensus.cc:374] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:49.198441 9968 raft_consensus.cc:729] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Initialized, Role: FOLLOWER
I20240502 14:16:49.198652 9968 consensus_queue.cc:260] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.198791 9968 raft_consensus.cc:388] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:49.198886 9968 raft_consensus.cc:482] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:49.198973 9968 raft_consensus.cc:3037] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:49.199283 9316 catalog_manager.cc:5484] T 542b5a718dc843bc8e9b623ba560d5a6 P 12fda910bc6142fb8f2d9aceffa40631 reported cstate change: term changed from 0 to 1, leader changed from <none> to 12fda910bc6142fb8f2d9aceffa40631 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "12fda910bc6142fb8f2d9aceffa40631" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:49.200868 9968 raft_consensus.cc:504] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.201092 9968 leader_election.cc:302] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 46d58e4957b949d8a1491d833eca1b6c; no voters:
I20240502 14:16:49.201326 9968 leader_election.cc:288] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:49.201454 9972 raft_consensus.cc:2781] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:49.201862 9968 ts_tablet_manager.cc:1398] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c: Time spent starting tablet: real 0.004s user 0.000s sys 0.004s
I20240502 14:16:49.201923 9972 raft_consensus.cc:686] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [term 1 LEADER]: Becoming Leader. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Running, Role: LEADER
I20240502 14:16:49.202407 9972 consensus_queue.cc:237] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.204975 9316 catalog_manager.cc:5484] T fdc2fee6e20d4e558f8f8366da152ab9 P 46d58e4957b949d8a1491d833eca1b6c reported cstate change: term changed from 0 to 1, leader changed from <none> to 46d58e4957b949d8a1491d833eca1b6c (127.27.220.130). New cstate: current_term: 1 leader_uuid: "46d58e4957b949d8a1491d833eca1b6c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:49.713757 9316 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36978:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:49.723627 9438 tablet_service.cc:1467] Processing CreateTablet for tablet 8132313e46444e69a0b2780a1dc3b958 (DEFAULT_TABLE table=rebalance_test_table_2 [id=1fcf04d379544371892190e4163336cb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:49.723819 9569 tablet_service.cc:1467] Processing CreateTablet for tablet 54c59ad9ce8542858e0a90f4a5f9d2e9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=1fcf04d379544371892190e4163336cb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:49.724125 9438 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8132313e46444e69a0b2780a1dc3b958. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:49.724234 9569 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 54c59ad9ce8542858e0a90f4a5f9d2e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:49.726830 9437 tablet_service.cc:1467] Processing CreateTablet for tablet 95ebd3a6c57a4ed097083aee6c7a7c11 (DEFAULT_TABLE table=rebalance_test_table_2 [id=1fcf04d379544371892190e4163336cb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:49.727166 9437 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 95ebd3a6c57a4ed097083aee6c7a7c11. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:49.730495 10007 tablet_bootstrap.cc:492] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c: Bootstrap starting.
I20240502 14:16:49.732196 10007 tablet_bootstrap.cc:654] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:49.737994 10009 tablet_bootstrap.cc:492] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631: Bootstrap starting.
I20240502 14:16:49.739445 10009 tablet_bootstrap.cc:654] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:49.740056 10007 tablet_bootstrap.cc:492] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c: No bootstrap required, opened a new log
I20240502 14:16:49.740218 10007 ts_tablet_manager.cc:1367] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.007s
I20240502 14:16:49.741080 10007 raft_consensus.cc:348] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.741319 10007 raft_consensus.cc:374] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:49.741425 10007 raft_consensus.cc:729] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Initialized, Role: FOLLOWER
I20240502 14:16:49.741693 10007 consensus_queue.cc:260] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.741429 10009 tablet_bootstrap.cc:492] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631: No bootstrap required, opened a new log
I20240502 14:16:49.741942 10007 raft_consensus.cc:388] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:49.741963 10009 ts_tablet_manager.cc:1367] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:16:49.742090 10007 raft_consensus.cc:482] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:49.742274 10007 raft_consensus.cc:3037] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:49.742833 10009 raft_consensus.cc:348] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.743048 10009 raft_consensus.cc:374] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:49.743175 10009 raft_consensus.cc:729] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Initialized, Role: FOLLOWER
I20240502 14:16:49.743479 10009 consensus_queue.cc:260] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.743726 10009 raft_consensus.cc:388] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:49.743822 10009 raft_consensus.cc:482] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:49.743959 10009 raft_consensus.cc:3037] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:49.744378 10007 raft_consensus.cc:504] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.744599 10007 leader_election.cc:302] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 46d58e4957b949d8a1491d833eca1b6c; no voters:
I20240502 14:16:49.744803 10007 leader_election.cc:288] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:49.744877 9972 raft_consensus.cc:2781] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:49.745285 9972 raft_consensus.cc:686] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [term 1 LEADER]: Becoming Leader. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Running, Role: LEADER
I20240502 14:16:49.745565 9972 consensus_queue.cc:237] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:49.746712 10009 raft_consensus.cc:504] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.746982 10009 leader_election.cc:302] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 12fda910bc6142fb8f2d9aceffa40631; no voters:
I20240502 14:16:49.747507 10009 leader_election.cc:288] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:49.747642 10014 raft_consensus.cc:2781] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:49.748051 9316 catalog_manager.cc:5484] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c reported cstate change: term changed from 0 to 1, leader changed from <none> to 46d58e4957b949d8a1491d833eca1b6c (127.27.220.130). New cstate: current_term: 1 leader_uuid: "46d58e4957b949d8a1491d833eca1b6c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:49.751739 10014 raft_consensus.cc:686] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 LEADER]: Becoming Leader. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Running, Role: LEADER
I20240502 14:16:49.752198 10014 consensus_queue.cc:237] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.754714 10007 ts_tablet_manager.cc:1398] T 54c59ad9ce8542858e0a90f4a5f9d2e9 P 46d58e4957b949d8a1491d833eca1b6c: Time spent starting tablet: real 0.014s user 0.009s sys 0.002s
I20240502 14:16:49.755529 10009 ts_tablet_manager.cc:1398] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631: Time spent starting tablet: real 0.013s user 0.007s sys 0.006s
I20240502 14:16:49.755905 10009 tablet_bootstrap.cc:492] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631: Bootstrap starting.
I20240502 14:16:49.758157 10009 tablet_bootstrap.cc:654] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:49.760455 10009 tablet_bootstrap.cc:492] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631: No bootstrap required, opened a new log
I20240502 14:16:49.760633 10009 ts_tablet_manager.cc:1367] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 14:16:49.761577 10009 raft_consensus.cc:348] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.761787 10009 raft_consensus.cc:374] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:49.761891 10009 raft_consensus.cc:729] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Initialized, Role: FOLLOWER
I20240502 14:16:49.762184 10009 consensus_queue.cc:260] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.762418 10009 raft_consensus.cc:388] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:49.762537 10009 raft_consensus.cc:482] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:49.762662 10009 raft_consensus.cc:3037] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:49.765787 10009 raft_consensus.cc:504] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.766130 10009 leader_election.cc:302] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 12fda910bc6142fb8f2d9aceffa40631; no voters:
I20240502 14:16:49.766497 10014 raft_consensus.cc:2781] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:49.766666 10014 raft_consensus.cc:686] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [term 1 LEADER]: Becoming Leader. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Running, Role: LEADER
I20240502 14:16:49.766878 10014 consensus_queue.cc:237] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:49.767271 10009 leader_election.cc:288] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:49.767158 9316 catalog_manager.cc:5484] T 95ebd3a6c57a4ed097083aee6c7a7c11 P 12fda910bc6142fb8f2d9aceffa40631 reported cstate change: term changed from 0 to 1, leader changed from <none> to 12fda910bc6142fb8f2d9aceffa40631 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "12fda910bc6142fb8f2d9aceffa40631" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:49.767664 10009 ts_tablet_manager.cc:1398] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631: Time spent starting tablet: real 0.007s user 0.000s sys 0.007s
I20240502 14:16:49.771420 9316 catalog_manager.cc:5484] T 8132313e46444e69a0b2780a1dc3b958 P 12fda910bc6142fb8f2d9aceffa40631 reported cstate change: term changed from 0 to 1, leader changed from <none> to 12fda910bc6142fb8f2d9aceffa40631 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "12fda910bc6142fb8f2d9aceffa40631" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:50.279655 9316 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36978:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 14:16:50.288141 9569 tablet_service.cc:1467] Processing CreateTablet for tablet 6caf364f01134b75928b763e83afc166 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f9484c0e759647e0ab768821bf0e1bd1]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:50.288317 9568 tablet_service.cc:1467] Processing CreateTablet for tablet 333af09c811346e4beabf8ca7d63a8a7 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f9484c0e759647e0ab768821bf0e1bd1]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:50.288653 9569 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6caf364f01134b75928b763e83afc166. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:50.288655 9437 tablet_service.cc:1467] Processing CreateTablet for tablet f55455533d4e43f98bc4ef38da3ae9af (DEFAULT_TABLE table=rebalance_test_table_3 [id=f9484c0e759647e0ab768821bf0e1bd1]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:16:50.288976 9568 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 333af09c811346e4beabf8ca7d63a8a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:50.289129 9437 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f55455533d4e43f98bc4ef38da3ae9af. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:50.295405 10045 tablet_bootstrap.cc:492] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631: Bootstrap starting.
I20240502 14:16:50.295725 10046 tablet_bootstrap.cc:492] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c: Bootstrap starting.
I20240502 14:16:50.297305 10045 tablet_bootstrap.cc:654] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:50.297607 10046 tablet_bootstrap.cc:654] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:50.299080 10045 tablet_bootstrap.cc:492] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631: No bootstrap required, opened a new log
I20240502 14:16:50.299208 10045 ts_tablet_manager.cc:1367] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:16:50.299257 10046 tablet_bootstrap.cc:492] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c: No bootstrap required, opened a new log
I20240502 14:16:50.299419 10046 ts_tablet_manager.cc:1367] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:16:50.299862 10045 raft_consensus.cc:348] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:50.300006 10045 raft_consensus.cc:374] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:50.300076 10045 raft_consensus.cc:729] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Initialized, Role: FOLLOWER
I20240502 14:16:50.300011 10046 raft_consensus.cc:348] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:50.300189 10046 raft_consensus.cc:374] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:50.300343 10046 raft_consensus.cc:729] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Initialized, Role: FOLLOWER
I20240502 14:16:50.300341 10045 consensus_queue.cc:260] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:50.300609 10045 raft_consensus.cc:388] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:50.300621 10046 consensus_queue.cc:260] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:50.300753 10045 raft_consensus.cc:482] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:50.300868 10046 raft_consensus.cc:388] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:50.300896 10045 raft_consensus.cc:3037] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:50.300971 10046 raft_consensus.cc:482] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:50.301084 10046 raft_consensus.cc:3037] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:50.302925 10045 raft_consensus.cc:504] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:50.303146 10045 leader_election.cc:302] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 12fda910bc6142fb8f2d9aceffa40631; no voters:
I20240502 14:16:50.303226 10046 raft_consensus.cc:504] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:50.303334 10045 leader_election.cc:288] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:50.303452 10014 raft_consensus.cc:2781] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:50.303544 10046 leader_election.cc:302] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 46d58e4957b949d8a1491d833eca1b6c; no voters:
I20240502 14:16:50.304102 10045 ts_tablet_manager.cc:1398] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631: Time spent starting tablet: real 0.005s user 0.003s sys 0.002s
I20240502 14:16:50.304450 10046 leader_election.cc:288] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:50.304533 10051 raft_consensus.cc:2781] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:50.306456 10051 raft_consensus.cc:686] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [term 1 LEADER]: Becoming Leader. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Running, Role: LEADER
I20240502 14:16:50.306826 10051 consensus_queue.cc:237] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:50.308122 10014 raft_consensus.cc:686] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [term 1 LEADER]: Becoming Leader. State: Replica: 12fda910bc6142fb8f2d9aceffa40631, State: Running, Role: LEADER
I20240502 14:16:50.308270 10046 ts_tablet_manager.cc:1398] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c: Time spent starting tablet: real 0.009s user 0.003s sys 0.005s
I20240502 14:16:50.308439 10014 consensus_queue.cc:237] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } }
I20240502 14:16:50.308683 10046 tablet_bootstrap.cc:492] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c: Bootstrap starting.
I20240502 14:16:50.309468 9316 catalog_manager.cc:5484] T 6caf364f01134b75928b763e83afc166 P 46d58e4957b949d8a1491d833eca1b6c reported cstate change: term changed from 0 to 1, leader changed from <none> to 46d58e4957b949d8a1491d833eca1b6c (127.27.220.130). New cstate: current_term: 1 leader_uuid: "46d58e4957b949d8a1491d833eca1b6c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:50.310361 10046 tablet_bootstrap.cc:654] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:50.315538 10046 tablet_bootstrap.cc:492] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c: No bootstrap required, opened a new log
I20240502 14:16:50.315779 10046 ts_tablet_manager.cc:1367] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.003s
I20240502 14:16:50.316628 10046 raft_consensus.cc:348] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:50.316589 9316 catalog_manager.cc:5484] T f55455533d4e43f98bc4ef38da3ae9af P 12fda910bc6142fb8f2d9aceffa40631 reported cstate change: term changed from 0 to 1, leader changed from <none> to 12fda910bc6142fb8f2d9aceffa40631 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "12fda910bc6142fb8f2d9aceffa40631" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "12fda910bc6142fb8f2d9aceffa40631" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37173 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:50.316949 10046 raft_consensus.cc:374] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:50.317039 10046 raft_consensus.cc:729] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Initialized, Role: FOLLOWER
I20240502 14:16:50.317350 10046 consensus_queue.cc:260] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:50.317548 10046 raft_consensus.cc:388] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:50.317651 10046 raft_consensus.cc:482] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:50.317763 10046 raft_consensus.cc:3037] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:50.320286 10046 raft_consensus.cc:504] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:50.320613 10046 leader_election.cc:302] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 46d58e4957b949d8a1491d833eca1b6c; no voters:
I20240502 14:16:50.320905 10046 leader_election.cc:288] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:50.321008 10052 raft_consensus.cc:2781] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:50.321214 10052 raft_consensus.cc:686] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [term 1 LEADER]: Becoming Leader. State: Replica: 46d58e4957b949d8a1491d833eca1b6c, State: Running, Role: LEADER
I20240502 14:16:50.321635 10052 consensus_queue.cc:237] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } }
I20240502 14:16:50.321929 10046 ts_tablet_manager.cc:1398] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c: Time spent starting tablet: real 0.006s user 0.001s sys 0.005s
W20240502 14:16:50.323506 9635 tablet.cc:2369] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:16:50.324425 9316 catalog_manager.cc:5484] T 333af09c811346e4beabf8ca7d63a8a7 P 46d58e4957b949d8a1491d833eca1b6c reported cstate change: term changed from 0 to 1, leader changed from <none> to 46d58e4957b949d8a1491d833eca1b6c (127.27.220.130). New cstate: current_term: 1 leader_uuid: "46d58e4957b949d8a1491d833eca1b6c" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "46d58e4957b949d8a1491d833eca1b6c" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 38941 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:50.841197 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:38377
--local_ip_for_outbound_sockets=127.27.220.131
--tserver_master_addrs=127.27.220.190:42681
--webserver_port=35879
--webserver_interface=127.27.220.131
--builtin_ntp_servers=127.27.220.148:46499
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:16:51.183390 10081 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:51.183691 10081 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:51.183821 10081 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:51.191082 10081 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:51.191207 10081 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:51.191365 10081 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:16:51.200196 10081 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:46499
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:38377
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=35879
--tserver_master_addrs=127.27.220.190:42681
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:51.201570 10081 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:51.203261 10081 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:51.214758 10086 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:51.215242 10087 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:51.215910 10089 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:51.216589 10081 server_base.cc:1034] running on GCE node
I20240502 14:16:51.217190 10081 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:51.218103 10081 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:51.219311 10081 hybrid_clock.cc:648] HybridClock initialized: now 1714659411219272 us; error 29 us; skew 500 ppm
I20240502 14:16:51.219626 10081 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:51.221827 10081 webserver.cc:460] Webserver started at http://127.27.220.131:35879/ using document root <none> and password file <none>
I20240502 14:16:51.222487 10081 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:51.222613 10081 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:51.227571 10081 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 14:16:51.230412 10095 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:51.231729 10081 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:16:51.231880 10081 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "5802fae32c644b9ba3baceb911d98967"
format_stamp: "Formatted at 2024-05-02 14:16:43 on dist-test-slave-j72s"
I20240502 14:16:51.232719 10081 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:51.248724 10081 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:51.249575 10081 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:51.249851 10081 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:51.250762 10081 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:51.252792 10102 ts_tablet_manager.cc:522] Loading tablet metadata (0/1 complete)
I20240502 14:16:51.257951 10081 ts_tablet_manager.cc:559] Loaded tablet metadata (1 total tablets, 1 live tablets)
I20240502 14:16:51.258050 10081 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.006s user 0.001s sys 0.000s
I20240502 14:16:51.258181 10081 ts_tablet_manager.cc:574] Registering tablets (0/1 complete)
I20240502 14:16:51.260931 10081 ts_tablet_manager.cc:590] Registered 1 tablets
I20240502 14:16:51.261035 10081 ts_tablet_manager.cc:569] Time spent register tablets: real 0.003s user 0.000s sys 0.000s
I20240502 14:16:51.261341 10102 tablet_bootstrap.cc:492] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Bootstrap starting.
I20240502 14:16:51.282843 10102 log.cc:824] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:51.287147 10102 tablet_bootstrap.cc:492] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Bootstrap replayed 1/1 log segments. Stats: ops{read=1 overwritten=0 applied=1 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:16:51.287956 10102 tablet_bootstrap.cc:492] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Bootstrap complete.
I20240502 14:16:51.289279 10102 ts_tablet_manager.cc:1367] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Time spent bootstrapping tablet: real 0.028s user 0.017s sys 0.011s
I20240502 14:16:51.298957 10102 raft_consensus.cc:348] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } }
I20240502 14:16:51.300491 10102 raft_consensus.cc:729] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: 5802fae32c644b9ba3baceb911d98967, State: Initialized, Role: FOLLOWER
I20240502 14:16:51.301403 10102 consensus_queue.cc:260] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 1, Last appended: 1.1, Last appended by leader: 1, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } }
I20240502 14:16:51.301803 10102 raft_consensus.cc:388] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 1 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:51.302008 10102 raft_consensus.cc:482] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 1 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:51.302268 10102 raft_consensus.cc:3037] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 1 FOLLOWER]: Advancing to term 2
I20240502 14:16:51.307636 10102 raft_consensus.cc:504] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } }
I20240502 14:16:51.308367 10102 leader_election.cc:302] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 5802fae32c644b9ba3baceb911d98967; no voters:
I20240502 14:16:51.309023 10102 leader_election.cc:288] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [CANDIDATE]: Term 2 election: Requested vote from peers
I20240502 14:16:51.309329 10209 raft_consensus.cc:2781] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 14:16:51.310870 10102 ts_tablet_manager.cc:1398] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967: Time spent starting tablet: real 0.021s user 0.008s sys 0.012s
I20240502 14:16:51.313584 10209 raft_consensus.cc:686] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [term 2 LEADER]: Becoming Leader. State: Replica: 5802fae32c644b9ba3baceb911d98967, State: Running, Role: LEADER
I20240502 14:16:51.314136 10081 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:38377
I20240502 14:16:51.314214 10212 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:38377 every 8 connection(s)
I20240502 14:16:51.314204 10209 consensus_queue.cc:237] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 1, Current term: 2, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } }
I20240502 14:16:51.315752 10081 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:16:51.324009 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 10081
I20240502 14:16:51.324779 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.132:38413
--local_ip_for_outbound_sockets=127.27.220.132
--tserver_master_addrs=127.27.220.190:42681
--webserver_port=45345
--webserver_interface=127.27.220.132
--builtin_ntp_servers=127.27.220.148:46499
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:16:51.329502 10214 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42681
I20240502 14:16:51.329810 10214 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:51.330734 10214 heartbeater.cc:511] Master 127.27.220.190:42681 requested a full tablet report, sending...
I20240502 14:16:51.333467 9316 ts_manager.cc:194] Re-registered known tserver with Master: 5802fae32c644b9ba3baceb911d98967 (127.27.220.131:38377)
I20240502 14:16:51.334041 9316 catalog_manager.cc:5484] T 73f5d7eb8a334aa78ca245efdc1e092f P 5802fae32c644b9ba3baceb911d98967 reported cstate change: term changed from 1 to 2. New cstate: current_term: 2 leader_uuid: "5802fae32c644b9ba3baceb911d98967" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "5802fae32c644b9ba3baceb911d98967" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38377 } health_report { overall_health: HEALTHY } } }
I20240502 14:16:51.338829 9316 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:40199
I20240502 14:16:51.341109 10214 heartbeater.cc:503] Master 127.27.220.190:42681 was elected leader, sending a full tablet report...
W20240502 14:16:51.660490 10222 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:51.660758 10222 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:51.660912 10222 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:51.667421 10222 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:51.667534 10222 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:51.667670 10222 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.132
I20240502 14:16:51.675644 10222 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:46499
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.132:38413
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.27.220.132
--webserver_port=45345
--tserver_master_addrs=127.27.220.190:42681
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.132
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:51.676725 10222 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:51.678325 10222 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:51.689107 10229 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:51.689782 10231 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:51.690483 10228 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:51.691135 10222 server_base.cc:1034] running on GCE node
I20240502 14:16:51.691709 10222 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:51.692592 10222 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:51.693783 10222 hybrid_clock.cc:648] HybridClock initialized: now 1714659411693744 us; error 36 us; skew 500 ppm
I20240502 14:16:51.694063 10222 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:51.695950 10222 webserver.cc:460] Webserver started at http://127.27.220.132:45345/ using document root <none> and password file <none>
I20240502 14:16:51.696645 10222 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:51.696794 10222 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:51.701510 10222 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.001s sys 0.002s
I20240502 14:16:51.703985 10237 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:51.705237 10222 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240502 14:16:51.705422 10222 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "782bc0b09bad4847afe30ebacd237298"
format_stamp: "Formatted at 2024-05-02 14:16:44 on dist-test-slave-j72s"
I20240502 14:16:51.706279 10222 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:51.722294 10222 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:51.722921 10222 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:51.723161 10222 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:51.724001 10222 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:51.725309 10222 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:51.725383 10222 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:51.725498 10222 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:51.725574 10222 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.001s
I20240502 14:16:51.759447 10222 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.132:38413
I20240502 14:16:51.759498 10349 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.132:38413 every 8 connection(s)
I20240502 14:16:51.760885 10222 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 14:16:51.768002 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 10222
I20240502 14:16:51.773531 10350 heartbeater.cc:347] Connected to a master server at 127.27.220.190:42681
I20240502 14:16:51.773875 10350 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:51.774856 10350 heartbeater.cc:511] Master 127.27.220.190:42681 requested a full tablet report, sending...
I20240502 14:16:51.776379 9316 ts_manager.cc:194] Re-registered known tserver with Master: 782bc0b09bad4847afe30ebacd237298 (127.27.220.132:38413)
I20240502 14:16:51.777137 9316 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.132:49977
I20240502 14:16:52.235545 9437 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:52.247218 10285 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:52.249053 10144 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:52.251367 9568 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:52.436296 9437 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:52.445611 10144 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 14:16:52.447909 9568 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:52.460386 10285 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:52.710834 9437 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:52.717065 10285 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:16:52.726475 9568 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:16:52.727512 10144 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
9a5481ee95bc467b88f3c0b3ca1cbf0e | 127.27.220.190:42681 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+----------------------+-------------------------
builtin_ntp_servers | 127.27.220.148:46499 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+----------------------+---------+----------+----------------+-----------------
12fda910bc6142fb8f2d9aceffa40631 | 127.27.220.129:37173 | HEALTHY | <none> | 6 | 0
46d58e4957b949d8a1491d833eca1b6c | 127.27.220.130:38941 | HEALTHY | <none> | 6 | 0
5802fae32c644b9ba3baceb911d98967 | 127.27.220.131:38377 | HEALTHY | <none> | 1 | 0
782bc0b09bad4847afe30ebacd237298 | 127.27.220.132:38413 | HEALTHY | <none> | 0 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 4
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 4 server(s) checked
local_ip_for_outbound_sockets | 127.27.220.129 | experimental | 127.27.220.129:37173
local_ip_for_outbound_sockets | 127.27.220.130 | experimental | 127.27.220.130:38941
local_ip_for_outbound_sockets | 127.27.220.131 | experimental | 127.27.220.131:38377
local_ip_for_outbound_sockets | 127.27.220.132 | experimental | 127.27.220.132:38413
never_fsync | true | unsafe,advanced | all 4 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 4 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 4 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 4 server(s) checked
server_dump_info_format | pb | hidden | all 4 server(s) checked
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.27.220.129:37173
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.27.220.130:38941
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.27.220.131:38377
server_dump_info_path | /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.27.220.132:38413
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+----------------------+-------------------------
builtin_ntp_servers | 127.27.220.148:46499 | all 4 server(s) checked
time_source | builtin | all 4 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 5 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 1 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 1
Median | 6
Third Quartile | 6
Maximum | 6
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 4
Tables | 5
Tablets | 13
Replicas | 13
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 14:16:52.779382 28530 log_verifier.cc:126] Checking tablet 333af09c811346e4beabf8ca7d63a8a7
I20240502 14:16:52.779984 10350 heartbeater.cc:503] Master 127.27.220.190:42681 was elected leader, sending a full tablet report...
I20240502 14:16:52.787341 28530 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 333af09c811346e4beabf8ca7d63a8a7
I20240502 14:16:52.787418 28530 log_verifier.cc:126] Checking tablet 451589dd12df44da9e64bf048329f841
I20240502 14:16:52.794831 28530 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 451589dd12df44da9e64bf048329f841
I20240502 14:16:52.794910 28530 log_verifier.cc:126] Checking tablet 542b5a718dc843bc8e9b623ba560d5a6
I20240502 14:16:52.802220 28530 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 542b5a718dc843bc8e9b623ba560d5a6
I20240502 14:16:52.802294 28530 log_verifier.cc:126] Checking tablet 54c59ad9ce8542858e0a90f4a5f9d2e9
I20240502 14:16:52.809482 28530 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 54c59ad9ce8542858e0a90f4a5f9d2e9
I20240502 14:16:52.809567 28530 log_verifier.cc:126] Checking tablet 6145e4ff8fd34366bd1a2eddc988eb0e
I20240502 14:16:52.817025 28530 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 6145e4ff8fd34366bd1a2eddc988eb0e
I20240502 14:16:52.817121 28530 log_verifier.cc:126] Checking tablet 6caf364f01134b75928b763e83afc166
I20240502 14:16:52.824525 28530 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 6caf364f01134b75928b763e83afc166
I20240502 14:16:52.824595 28530 log_verifier.cc:126] Checking tablet 6cdbded6369f45e397c214e368c5aed6
I20240502 14:16:52.831696 28530 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 6cdbded6369f45e397c214e368c5aed6
I20240502 14:16:52.831768 28530 log_verifier.cc:126] Checking tablet 73f5d7eb8a334aa78ca245efdc1e092f
I20240502 14:16:52.839190 28530 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 73f5d7eb8a334aa78ca245efdc1e092f
I20240502 14:16:52.839264 28530 log_verifier.cc:126] Checking tablet 8132313e46444e69a0b2780a1dc3b958
I20240502 14:16:52.846601 28530 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 8132313e46444e69a0b2780a1dc3b958
I20240502 14:16:52.846674 28530 log_verifier.cc:126] Checking tablet 95ebd3a6c57a4ed097083aee6c7a7c11
I20240502 14:16:52.853761 28530 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 95ebd3a6c57a4ed097083aee6c7a7c11
I20240502 14:16:52.853827 28530 log_verifier.cc:126] Checking tablet c245d4a31c5440c886c443d91d20c1ff
I20240502 14:16:52.860906 28530 log_verifier.cc:177] Verified matching terms for 1 ops in tablet c245d4a31c5440c886c443d91d20c1ff
I20240502 14:16:52.860975 28530 log_verifier.cc:126] Checking tablet f55455533d4e43f98bc4ef38da3ae9af
I20240502 14:16:52.868430 28530 log_verifier.cc:177] Verified matching terms for 1 ops in tablet f55455533d4e43f98bc4ef38da3ae9af
I20240502 14:16:52.868505 28530 log_verifier.cc:126] Checking tablet fdc2fee6e20d4e558f8f8366da152ab9
I20240502 14:16:52.876391 28530 log_verifier.cc:177] Verified matching terms for 2 ops in tablet fdc2fee6e20d4e558f8f8366da152ab9
I20240502 14:16:52.877300 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 9376
I20240502 14:16:52.878669 9498 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:52.976696 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 9376
I20240502 14:16:52.996096 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 9507
I20240502 14:16:52.997108 9628 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:53.095353 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 9507
I20240502 14:16:53.114971 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 10081
I20240502 14:16:53.121069 10205 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:53.207834 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 10081
I20240502 14:16:53.225724 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 10222
I20240502 14:16:53.231562 10344 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:53.318831 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 10222
I20240502 14:16:53.337284 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 9285
I20240502 14:16:53.338449 9346 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:53.457500 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 9285
2024-05-02T14:16:53Z chronyd exiting
[ OK ] RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove/5 (11427 ms)
[----------] 1 test from RebalancerAndSingleReplicaTablets (11427 ms total)
[----------] 2 tests from LocationAwareRebalancingParamTest
[ RUN ] LocationAwareRebalancingParamTest.Rebalance/idx7_rf2_t3_l_1_2_1
I20240502 14:16:53.516075 28530 test_util.cc:268] Using random seed: 1094770450
I20240502 14:16:53.518124 28530 ts_itest-base.cc:112] Starting cluster with:
I20240502 14:16:53.518193 28530 ts_itest-base.cc:113] --------------
I20240502 14:16:53.518239 28530 ts_itest-base.cc:114] 4 tablet servers
I20240502 14:16:53.518287 28530 ts_itest-base.cc:115] 2 replicas per TS
I20240502 14:16:53.518342 28530 ts_itest-base.cc:116] --------------
2024-05-02T14:16:53Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T14:16:53Z Disabled control of system clock
I20240502 14:16:53.569085 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:44381
--webserver_interface=127.27.220.190
--webserver_port=0
--builtin_ntp_servers=127.27.220.148:34559
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.27.220.190:44381
--location_mapping_cmd=/tmp/dist-test-taskhB7nWc/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/location-assignment.state --map /A:1 --map /B:2 --map /C:1
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false
--master_client_location_assignment_enabled=false
--allow_unsafe_replication_factor with env {}
W20240502 14:16:53.895234 10465 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 14:16:53.895599 10465 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:53.895706 10465 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:53.895817 10465 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:53.903204 10465 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:53.903358 10465 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 14:16:53.903489 10465 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:53.903576 10465 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 14:16:53.903674 10465 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 14:16:53.912957 10465 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:34559
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-taskhB7nWc/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/location-assignment.state --map /A:1 --map /B:2 --map /C:1
--ipki_ca_key_size=768
--master_addresses=127.27.220.190:44381
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:44381
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.27.220.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:53.914319 10465 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:53.916102 10465 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:53.926824 10470 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:53.927187 10471 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:53.927649 10473 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:53.928316 10465 server_base.cc:1034] running on GCE node
I20240502 14:16:53.929056 10465 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:53.930408 10465 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:53.931618 10465 hybrid_clock.cc:648] HybridClock initialized: now 1714659413931562 us; error 27 us; skew 500 ppm
I20240502 14:16:53.931914 10465 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:53.934126 10465 webserver.cc:460] Webserver started at http://127.27.220.190:43773/ using document root <none> and password file <none>
I20240502 14:16:53.934871 10465 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:53.934996 10465 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:53.935289 10465 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:53.937947 10465 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "f1869cf667264056935fa98531707cee"
format_stamp: "Formatted at 2024-05-02 14:16:53 on dist-test-slave-j72s"
I20240502 14:16:53.938711 10465 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "f1869cf667264056935fa98531707cee"
format_stamp: "Formatted at 2024-05-02 14:16:53 on dist-test-slave-j72s"
I20240502 14:16:53.943902 10465 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 14:16:53.947270 10479 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:53.948592 10465 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240502 14:16:53.948755 10465 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
uuid: "f1869cf667264056935fa98531707cee"
format_stamp: "Formatted at 2024-05-02 14:16:53 on dist-test-slave-j72s"
I20240502 14:16:53.948942 10465 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:53.976640 10465 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:53.977721 10465 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:53.978051 10465 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:53.996424 10465 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.190:44381
I20240502 14:16:53.996501 10530 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.190:44381 every 8 connection(s)
I20240502 14:16:53.997874 10465 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 14:16:54.001715 10531 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:16:54.007048 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 10465
I20240502 14:16:54.007303 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 14:16:54.012559 10531 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee: Bootstrap starting.
I20240502 14:16:54.016711 10531 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee: Neither blocks nor log segments found. Creating new log.
I20240502 14:16:54.017959 10531 log.cc:824] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee: Log is configured to *not* fsync() on all Append() calls
I20240502 14:16:54.021057 10531 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee: No bootstrap required, opened a new log
I20240502 14:16:54.027858 10531 raft_consensus.cc:348] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f1869cf667264056935fa98531707cee" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 44381 } }
I20240502 14:16:54.028265 10531 raft_consensus.cc:374] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:16:54.028383 10531 raft_consensus.cc:729] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f1869cf667264056935fa98531707cee, State: Initialized, Role: FOLLOWER
I20240502 14:16:54.028970 10531 consensus_queue.cc:260] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f1869cf667264056935fa98531707cee" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 44381 } }
I20240502 14:16:54.029187 10531 raft_consensus.cc:388] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:16:54.029325 10531 raft_consensus.cc:482] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:16:54.029466 10531 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:16:54.031572 10531 raft_consensus.cc:504] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f1869cf667264056935fa98531707cee" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 44381 } }
I20240502 14:16:54.032172 10531 leader_election.cc:302] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f1869cf667264056935fa98531707cee; no voters:
I20240502 14:16:54.032828 10531 leader_election.cc:288] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:16:54.032959 10536 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:16:54.033764 10536 raft_consensus.cc:686] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [term 1 LEADER]: Becoming Leader. State: Replica: f1869cf667264056935fa98531707cee, State: Running, Role: LEADER
I20240502 14:16:54.034289 10536 consensus_queue.cc:237] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f1869cf667264056935fa98531707cee" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 44381 } }
I20240502 14:16:54.034889 10531 sys_catalog.cc:564] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [sys.catalog]: configured and running, proceeding with master startup.
I20240502 14:16:54.037251 10537 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "f1869cf667264056935fa98531707cee" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f1869cf667264056935fa98531707cee" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 44381 } } }
I20240502 14:16:54.037681 10537 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [sys.catalog]: This master's current role is: LEADER
I20240502 14:16:54.037761 10538 sys_catalog.cc:455] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [sys.catalog]: SysCatalogTable state changed. Reason: New leader f1869cf667264056935fa98531707cee. Latest consensus state: current_term: 1 leader_uuid: "f1869cf667264056935fa98531707cee" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f1869cf667264056935fa98531707cee" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 44381 } } }
I20240502 14:16:54.038060 10538 sys_catalog.cc:458] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee [sys.catalog]: This master's current role is: LEADER
I20240502 14:16:54.042768 10543 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 14:16:54.047425 10543 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 14:16:54.055965 10543 catalog_manager.cc:1347] Generated new cluster ID: a9ed3256023c4cc9b80ce6ee0edf8e0a
I20240502 14:16:54.056089 10543 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 14:16:54.069231 10543 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 14:16:54.070542 10543 catalog_manager.cc:1504] Loading token signing keys...
I20240502 14:16:54.080705 10543 catalog_manager.cc:5857] T 00000000000000000000000000000000 P f1869cf667264056935fa98531707cee: Generated new TSK 0
I20240502 14:16:54.081482 10543 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 14:16:54.089270 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.129:0
--local_ip_for_outbound_sockets=127.27.220.129
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:44381
--builtin_ntp_servers=127.27.220.148:34559
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:16:54.426446 10555 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:54.426708 10555 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:54.426836 10555 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:54.434195 10555 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:54.434324 10555 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:54.434475 10555 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.129
I20240502 14:16:54.443017 10555 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:34559
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:44381
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.129
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:54.444324 10555 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:54.446055 10555 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:54.457823 10560 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:54.458283 10563 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:54.458793 10561 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:54.459214 10555 server_base.cc:1034] running on GCE node
I20240502 14:16:54.459815 10555 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:54.460798 10555 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:54.462006 10555 hybrid_clock.cc:648] HybridClock initialized: now 1714659414461948 us; error 47 us; skew 500 ppm
I20240502 14:16:54.462333 10555 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:54.468292 10555 webserver.cc:460] Webserver started at http://127.27.220.129:46743/ using document root <none> and password file <none>
I20240502 14:16:54.469059 10555 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:54.469182 10555 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:54.469498 10555 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:54.472254 10555 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "ced0bd81140c45c19192ed2611f0d58a"
format_stamp: "Formatted at 2024-05-02 14:16:54 on dist-test-slave-j72s"
I20240502 14:16:54.473011 10555 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "ced0bd81140c45c19192ed2611f0d58a"
format_stamp: "Formatted at 2024-05-02 14:16:54 on dist-test-slave-j72s"
I20240502 14:16:54.478119 10555 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.002s
I20240502 14:16:54.481361 10569 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:54.482673 10555 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 14:16:54.482832 10555 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "ced0bd81140c45c19192ed2611f0d58a"
format_stamp: "Formatted at 2024-05-02 14:16:54 on dist-test-slave-j72s"
I20240502 14:16:54.483026 10555 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:54.509814 10555 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:54.510619 10555 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:54.510886 10555 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:54.511816 10555 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:54.513306 10555 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:54.513408 10555 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:54.513535 10555 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:54.513615 10555 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.001s
I20240502 14:16:54.546757 10555 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.129:37689
I20240502 14:16:54.546813 10681 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.129:37689 every 8 connection(s)
I20240502 14:16:54.548022 10555 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 14:16:54.552196 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 10555
I20240502 14:16:54.552435 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 14:16:54.556099 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.130:0
--local_ip_for_outbound_sockets=127.27.220.130
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:44381
--builtin_ntp_servers=127.27.220.148:34559
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:16:54.561288 10682 heartbeater.cc:347] Connected to a master server at 127.27.220.190:44381
I20240502 14:16:54.561635 10682 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:54.562561 10682 heartbeater.cc:511] Master 127.27.220.190:44381 requested a full tablet report, sending...
I20240502 14:16:54.624182 10496 ts_manager.cc:194] Registered new tserver with Master: ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:16:54.625949 10496 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.129:34635
W20240502 14:16:54.893182 10686 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:54.893488 10686 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:54.893662 10686 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:54.901157 10686 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:54.901320 10686 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:54.901558 10686 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.130
I20240502 14:16:54.910665 10686 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:34559
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:44381
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.130
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:54.911960 10686 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:54.913731 10686 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:54.925688 10695 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:54.925904 10692 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:54.926360 10693 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:54.926920 10686 server_base.cc:1034] running on GCE node
I20240502 14:16:54.927487 10686 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:54.928318 10686 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:54.929567 10686 hybrid_clock.cc:648] HybridClock initialized: now 1714659414929496 us; error 33 us; skew 500 ppm
I20240502 14:16:54.929847 10686 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:54.932060 10686 webserver.cc:460] Webserver started at http://127.27.220.130:43281/ using document root <none> and password file <none>
I20240502 14:16:54.932765 10686 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:54.932880 10686 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:54.933185 10686 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:54.935787 10686 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "3be4f0789bca4f0483a4d3df81a8b0c3"
format_stamp: "Formatted at 2024-05-02 14:16:54 on dist-test-slave-j72s"
I20240502 14:16:54.936586 10686 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "3be4f0789bca4f0483a4d3df81a8b0c3"
format_stamp: "Formatted at 2024-05-02 14:16:54 on dist-test-slave-j72s"
I20240502 14:16:54.941473 10686 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.004s
I20240502 14:16:54.944749 10701 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:54.946053 10686 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:16:54.946216 10686 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "3be4f0789bca4f0483a4d3df81a8b0c3"
format_stamp: "Formatted at 2024-05-02 14:16:54 on dist-test-slave-j72s"
I20240502 14:16:54.946425 10686 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:54.972115 10686 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:54.972963 10686 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:54.973271 10686 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:54.974148 10686 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:54.975468 10686 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:54.975565 10686 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:54.975643 10686 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:54.975695 10686 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:55.008311 10686 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.130:33583
I20240502 14:16:55.008349 10813 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.130:33583 every 8 connection(s)
I20240502 14:16:55.009538 10686 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 14:16:55.019109 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 10686
I20240502 14:16:55.019291 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 14:16:55.021596 10814 heartbeater.cc:347] Connected to a master server at 127.27.220.190:44381
I20240502 14:16:55.021867 10814 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:55.022604 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:0
--local_ip_for_outbound_sockets=127.27.220.131
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:44381
--builtin_ntp_servers=127.27.220.148:34559
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:16:55.022636 10814 heartbeater.cc:511] Master 127.27.220.190:44381 requested a full tablet report, sending...
I20240502 14:16:55.068030 10496 ts_manager.cc:194] Registered new tserver with Master: 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130:33583)
I20240502 14:16:55.068895 10496 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.130:41333
W20240502 14:16:55.350056 10818 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:55.350383 10818 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:55.350524 10818 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:55.358063 10818 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:55.358229 10818 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:55.358409 10818 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:16:55.367461 10818 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:34559
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:44381
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:55.368798 10818 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:55.370467 10818 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:55.382165 10826 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:55.382293 10823 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:55.382871 10824 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:55.383550 10818 server_base.cc:1034] running on GCE node
I20240502 14:16:55.384052 10818 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:55.384862 10818 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:55.386054 10818 hybrid_clock.cc:648] HybridClock initialized: now 1714659415386016 us; error 36 us; skew 500 ppm
I20240502 14:16:55.386373 10818 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:55.388604 10818 webserver.cc:460] Webserver started at http://127.27.220.131:33383/ using document root <none> and password file <none>
I20240502 14:16:55.389292 10818 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:55.389407 10818 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:55.389739 10818 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:55.392251 10818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "4224d6603b434d6cb7cf3b58a86a041f"
format_stamp: "Formatted at 2024-05-02 14:16:55 on dist-test-slave-j72s"
I20240502 14:16:55.392971 10818 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "4224d6603b434d6cb7cf3b58a86a041f"
format_stamp: "Formatted at 2024-05-02 14:16:55 on dist-test-slave-j72s"
I20240502 14:16:55.398011 10818 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.005s
I20240502 14:16:55.401166 10832 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:55.402479 10818 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:16:55.402622 10818 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "4224d6603b434d6cb7cf3b58a86a041f"
format_stamp: "Formatted at 2024-05-02 14:16:55 on dist-test-slave-j72s"
I20240502 14:16:55.402810 10818 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:55.414791 10818 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:55.415596 10818 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:55.415894 10818 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:55.416884 10818 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:55.418363 10818 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:55.418466 10818 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:55.418582 10818 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:55.418653 10818 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:55.449045 10818 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:38943
I20240502 14:16:55.449098 10944 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:38943 every 8 connection(s)
I20240502 14:16:55.450486 10818 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:16:55.453928 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 10818
I20240502 14:16:55.454139 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 14:16:55.457500 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.132:0
--local_ip_for_outbound_sockets=127.27.220.132
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:44381
--builtin_ntp_servers=127.27.220.148:34559
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:16:55.464260 10945 heartbeater.cc:347] Connected to a master server at 127.27.220.190:44381
I20240502 14:16:55.464583 10945 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:55.465425 10945 heartbeater.cc:511] Master 127.27.220.190:44381 requested a full tablet report, sending...
I20240502 14:16:55.510721 10496 ts_manager.cc:194] Registered new tserver with Master: 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131:38943)
I20240502 14:16:55.511790 10496 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:54413
I20240502 14:16:55.628849 10682 heartbeater.cc:503] Master 127.27.220.190:44381 was elected leader, sending a full tablet report...
W20240502 14:16:55.795347 10948 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:16:55.795598 10948 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:16:55.795677 10948 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:16:55.802455 10948 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:16:55.802559 10948 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:16:55.802659 10948 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.132
I20240502 14:16:55.810712 10948 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:34559
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:44381
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.132
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:16:55.811879 10948 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:16:55.813503 10948 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:16:55.825278 10954 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:55.825431 10955 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:16:55.826027 10957 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:16:55.826555 10948 server_base.cc:1034] running on GCE node
I20240502 14:16:55.827054 10948 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:16:55.827811 10948 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:16:55.829012 10948 hybrid_clock.cc:648] HybridClock initialized: now 1714659415828961 us; error 40 us; skew 500 ppm
I20240502 14:16:55.829324 10948 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:16:55.831416 10948 webserver.cc:460] Webserver started at http://127.27.220.132:42079/ using document root <none> and password file <none>
I20240502 14:16:55.832139 10948 fs_manager.cc:359] Metadata directory not provided
I20240502 14:16:55.832239 10948 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:16:55.832545 10948 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:16:55.837031 10948 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "76827763179e465a92298cea85e53f2b"
format_stamp: "Formatted at 2024-05-02 14:16:55 on dist-test-slave-j72s"
I20240502 14:16:55.837792 10948 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "76827763179e465a92298cea85e53f2b"
format_stamp: "Formatted at 2024-05-02 14:16:55 on dist-test-slave-j72s"
I20240502 14:16:55.842629 10948 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
I20240502 14:16:55.845731 10963 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:55.846948 10948 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
I20240502 14:16:55.847075 10948 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "76827763179e465a92298cea85e53f2b"
format_stamp: "Formatted at 2024-05-02 14:16:55 on dist-test-slave-j72s"
I20240502 14:16:55.847242 10948 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:16:55.865672 10948 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:16:55.866356 10948 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:16:55.866567 10948 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:16:55.867333 10948 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:16:55.868641 10948 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:16:55.868719 10948 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:55.868808 10948 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:16:55.868881 10948 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:16:55.899680 10948 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.132:35055
I20240502 14:16:55.899763 11075 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.132:35055 every 8 connection(s)
I20240502 14:16:55.901021 10948 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 14:16:55.909839 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 10948
I20240502 14:16:55.910020 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 14:16:55.914151 11076 heartbeater.cc:347] Connected to a master server at 127.27.220.190:44381
I20240502 14:16:55.914471 11076 heartbeater.cc:464] Registering TS with master...
I20240502 14:16:55.915249 11076 heartbeater.cc:511] Master 127.27.220.190:44381 requested a full tablet report, sending...
I20240502 14:16:55.960623 10496 ts_manager.cc:194] Registered new tserver with Master: 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:16:55.961679 10496 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.132:56057
I20240502 14:16:55.964375 28530 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 14:16:56.012940 28530 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 14:16:56.013108 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 10686
I20240502 14:16:56.021878 10809 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:56.101590 10814 heartbeater.cc:503] Master 127.27.220.190:44381 was elected leader, sending a full tablet report...
I20240502 14:16:56.102818 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 10686
W20240502 14:16:56.120265 10480 connection.cc:535] server connection from 127.27.220.130:41333 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:16:56.120498 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 10818
I20240502 14:16:56.128538 10940 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:16:56.208410 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 10818
I20240502 14:16:56.965184 11076 heartbeater.cc:503] Master 127.27.220.190:44381 was elected leader, sending a full tablet report...
I20240502 14:16:59.977854 10496 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55026:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:16:59.984884 10496 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240502 14:17:00.015620 11011 tablet_service.cc:1467] Processing CreateTablet for tablet fd038ee98b7f404abfe537b8e44d9baa (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e56c7cd46804033b4ea083d1730d37b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:00.016237 11009 tablet_service.cc:1467] Processing CreateTablet for tablet e4876c504a97477cb27fb7f6c4ae12b6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e56c7cd46804033b4ea083d1730d37b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:00.017205 11009 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e4876c504a97477cb27fb7f6c4ae12b6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:00.015501 11010 tablet_service.cc:1467] Processing CreateTablet for tablet ab98a42082f34b839684b25d6386898b (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e56c7cd46804033b4ea083d1730d37b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:00.017758 11011 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fd038ee98b7f404abfe537b8e44d9baa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:00.018153 11010 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ab98a42082f34b839684b25d6386898b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:00.021145 10616 tablet_service.cc:1467] Processing CreateTablet for tablet ab98a42082f34b839684b25d6386898b (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e56c7cd46804033b4ea083d1730d37b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:00.021028 10617 tablet_service.cc:1467] Processing CreateTablet for tablet fd038ee98b7f404abfe537b8e44d9baa (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e56c7cd46804033b4ea083d1730d37b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:00.022195 10615 tablet_service.cc:1467] Processing CreateTablet for tablet e4876c504a97477cb27fb7f6c4ae12b6 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e56c7cd46804033b4ea083d1730d37b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:00.022630 10617 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fd038ee98b7f404abfe537b8e44d9baa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:00.025677 10615 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e4876c504a97477cb27fb7f6c4ae12b6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:00.026257 10616 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ab98a42082f34b839684b25d6386898b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:00.038022 11099 tablet_bootstrap.cc:492] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:00.042586 11099 tablet_bootstrap.cc:654] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:00.042752 11098 tablet_bootstrap.cc:492] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:00.043721 11099 log.cc:824] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:00.045605 11098 tablet_bootstrap.cc:654] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:00.046408 11099 tablet_bootstrap.cc:492] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:00.046674 11099 ts_tablet_manager.cc:1367] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.009s user 0.008s sys 0.000s
I20240502 14:17:00.046815 11098 log.cc:824] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:00.049340 11098 tablet_bootstrap.cc:492] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:00.049623 11098 ts_tablet_manager.cc:1367] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.005s
I20240502 14:17:00.053279 11099 raft_consensus.cc:348] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:00.053661 11099 raft_consensus.cc:374] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:00.053769 11099 raft_consensus.cc:729] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:00.054391 11099 consensus_queue.cc:260] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:00.055948 11099 ts_tablet_manager.cc:1398] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.009s user 0.006s sys 0.003s
I20240502 14:17:00.056027 11098 raft_consensus.cc:348] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.056352 11099 tablet_bootstrap.cc:492] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:00.056383 11098 raft_consensus.cc:374] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:00.056478 11098 raft_consensus.cc:729] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:00.057154 11098 consensus_queue.cc:260] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.058043 11099 tablet_bootstrap.cc:654] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:00.059770 11099 tablet_bootstrap.cc:492] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:00.059900 11099 ts_tablet_manager.cc:1367] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:17:00.060171 11098 ts_tablet_manager.cc:1398] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.010s user 0.007s sys 0.002s
I20240502 14:17:00.060860 11099 raft_consensus.cc:348] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.061055 11099 raft_consensus.cc:374] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:00.061161 11099 raft_consensus.cc:729] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:00.061412 11099 consensus_queue.cc:260] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.061856 11099 ts_tablet_manager.cc:1398] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:00.062136 11099 tablet_bootstrap.cc:492] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:00.062916 11098 tablet_bootstrap.cc:492] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:00.064399 11098 tablet_bootstrap.cc:654] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:00.064469 11099 tablet_bootstrap.cc:654] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:00.066905 11099 tablet_bootstrap.cc:492] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:00.067057 11098 tablet_bootstrap.cc:492] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:00.067081 11099 ts_tablet_manager.cc:1367] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.001s
I20240502 14:17:00.067200 11098 ts_tablet_manager.cc:1367] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:17:00.067770 11099 raft_consensus.cc:348] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.067976 11099 raft_consensus.cc:374] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:00.068059 11099 raft_consensus.cc:729] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:00.068126 11098 raft_consensus.cc:348] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:00.068289 11098 raft_consensus.cc:374] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:00.068275 11099 consensus_queue.cc:260] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.068374 11098 raft_consensus.cc:729] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:00.068599 11098 consensus_queue.cc:260] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:00.068766 11099 ts_tablet_manager.cc:1398] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:00.069084 11098 ts_tablet_manager.cc:1398] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:00.069331 11098 tablet_bootstrap.cc:492] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:00.070508 11098 tablet_bootstrap.cc:654] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:00.072916 11098 tablet_bootstrap.cc:492] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:00.073058 11098 ts_tablet_manager.cc:1367] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:17:00.073611 11098 raft_consensus.cc:348] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.073793 11098 raft_consensus.cc:374] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:00.073872 11098 raft_consensus.cc:729] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:00.074071 11098 consensus_queue.cc:260] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.074512 11098 ts_tablet_manager.cc:1398] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 14:17:00.114212 11103 raft_consensus.cc:482] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:00.114459 11103 raft_consensus.cc:504] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:00.115911 11103 leader_election.cc:288] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:00.122205 10637 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fd038ee98b7f404abfe537b8e44d9baa" candidate_uuid: "76827763179e465a92298cea85e53f2b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced0bd81140c45c19192ed2611f0d58a" is_pre_election: true
I20240502 14:17:00.122689 10637 raft_consensus.cc:2454] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 76827763179e465a92298cea85e53f2b in term 0.
I20240502 14:17:00.123253 10966 leader_election.cc:302] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:00.123595 11103 raft_consensus.cc:2781] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:00.123718 11103 raft_consensus.cc:482] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:00.123822 11103 raft_consensus.cc:3037] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:00.125782 11103 raft_consensus.cc:504] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:00.126199 11103 leader_election.cc:288] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 election: Requested vote from peers ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:00.126626 10637 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fd038ee98b7f404abfe537b8e44d9baa" candidate_uuid: "76827763179e465a92298cea85e53f2b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced0bd81140c45c19192ed2611f0d58a"
I20240502 14:17:00.126776 10637 raft_consensus.cc:3037] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:00.128684 10637 raft_consensus.cc:2454] T fd038ee98b7f404abfe537b8e44d9baa P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 76827763179e465a92298cea85e53f2b in term 1.
I20240502 14:17:00.129050 10966 leader_election.cc:302] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:00.129292 11103 raft_consensus.cc:2781] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:00.129750 11103 raft_consensus.cc:686] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Becoming Leader. State: Replica: 76827763179e465a92298cea85e53f2b, State: Running, Role: LEADER
I20240502 14:17:00.130159 11103 consensus_queue.cc:237] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:00.134282 10495 catalog_manager.cc:5484] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b reported cstate change: term changed from 0 to 1, leader changed from <none> to 76827763179e465a92298cea85e53f2b (127.27.220.132). New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: UNKNOWN } } }
W20240502 14:17:00.155699 11077 tablet.cc:2369] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:17:00.217453 11103 raft_consensus.cc:482] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:00.217710 11103 raft_consensus.cc:504] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.218197 11103 leader_election.cc:288] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:00.218798 10637 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e4876c504a97477cb27fb7f6c4ae12b6" candidate_uuid: "76827763179e465a92298cea85e53f2b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced0bd81140c45c19192ed2611f0d58a" is_pre_election: true
I20240502 14:17:00.219017 10637 raft_consensus.cc:2454] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 76827763179e465a92298cea85e53f2b in term 0.
I20240502 14:17:00.219399 10966 leader_election.cc:302] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:00.219662 11103 raft_consensus.cc:2781] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:00.219741 11103 raft_consensus.cc:482] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:00.219831 11103 raft_consensus.cc:3037] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:00.221752 11103 raft_consensus.cc:504] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.222169 11103 leader_election.cc:288] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 election: Requested vote from peers ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:00.222615 10637 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e4876c504a97477cb27fb7f6c4ae12b6" candidate_uuid: "76827763179e465a92298cea85e53f2b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced0bd81140c45c19192ed2611f0d58a"
I20240502 14:17:00.222805 10637 raft_consensus.cc:3037] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:00.224539 10637 raft_consensus.cc:2454] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 76827763179e465a92298cea85e53f2b in term 1.
I20240502 14:17:00.224856 10966 leader_election.cc:302] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:00.225095 11103 raft_consensus.cc:2781] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:00.225272 11103 raft_consensus.cc:686] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Becoming Leader. State: Replica: 76827763179e465a92298cea85e53f2b, State: Running, Role: LEADER
I20240502 14:17:00.225541 11103 consensus_queue.cc:237] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.227687 10495 catalog_manager.cc:5484] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b reported cstate change: term changed from 0 to 1, leader changed from <none> to 76827763179e465a92298cea85e53f2b (127.27.220.132). New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } }
W20240502 14:17:00.304180 10683 tablet.cc:2369] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:17:00.310309 11102 raft_consensus.cc:482] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:00.310534 11102 raft_consensus.cc:504] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.312039 11102 leader_election.cc:288] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:00.318424 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ab98a42082f34b839684b25d6386898b" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b" is_pre_election: true
I20240502 14:17:00.318815 11031 raft_consensus.cc:2454] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 0.
I20240502 14:17:00.319325 10572 leader_election.cc:302] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:00.319662 11102 raft_consensus.cc:2781] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:00.319777 11102 raft_consensus.cc:482] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:00.319862 11102 raft_consensus.cc:3037] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:00.321877 11102 raft_consensus.cc:504] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.322355 11102 leader_election.cc:288] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Requested vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:00.322790 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ab98a42082f34b839684b25d6386898b" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b"
I20240502 14:17:00.322939 11031 raft_consensus.cc:3037] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:00.324874 11031 raft_consensus.cc:2454] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 1.
I20240502 14:17:00.325208 10572 leader_election.cc:302] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:00.325492 11102 raft_consensus.cc:2781] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:00.325906 11102 raft_consensus.cc:686] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Becoming Leader. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Running, Role: LEADER
I20240502 14:17:00.326320 11102 consensus_queue.cc:237] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:00.330415 10495 catalog_manager.cc:5484] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a reported cstate change: term changed from 0 to 1, leader changed from <none> to ced0bd81140c45c19192ed2611f0d58a (127.27.220.129). New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: UNKNOWN } } }
W20240502 14:17:00.726821 11115 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:00.727131 11115 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:17:00.733553 11119 consensus_queue.cc:1035] T fd038ee98b7f404abfe537b8e44d9baa P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:00.736115 11115 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:17:00.753455 11121 consensus_queue.cc:1035] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_0'
I20240502 14:17:00.813968 11102 consensus_queue.cc:1035] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 206.685 ms
time per row: 0.103343 ms
I20240502 14:17:01.091882 10495 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55026:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:01.092708 10495 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_1 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240502 14:17:01.100355 11010 tablet_service.cc:1467] Processing CreateTablet for tablet e8980b56d2b04a2682f9e8c1b2c5e875 (DEFAULT_TABLE table=rebalance_test_table_1 [id=672006a2715b45439494a055f37d9484]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:01.100594 10615 tablet_service.cc:1467] Processing CreateTablet for tablet e8980b56d2b04a2682f9e8c1b2c5e875 (DEFAULT_TABLE table=rebalance_test_table_1 [id=672006a2715b45439494a055f37d9484]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:01.100857 11010 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8980b56d2b04a2682f9e8c1b2c5e875. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:01.101052 10615 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8980b56d2b04a2682f9e8c1b2c5e875. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:01.101086 10616 tablet_service.cc:1467] Processing CreateTablet for tablet d254a1ad11914371a77003337335011b (DEFAULT_TABLE table=rebalance_test_table_1 [id=672006a2715b45439494a055f37d9484]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:01.101495 10616 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d254a1ad11914371a77003337335011b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:01.101581 10617 tablet_service.cc:1467] Processing CreateTablet for tablet 1b42bec0240f4547b2acdd6c8269d0eb (DEFAULT_TABLE table=rebalance_test_table_1 [id=672006a2715b45439494a055f37d9484]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:01.101936 10617 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1b42bec0240f4547b2acdd6c8269d0eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:01.102212 11009 tablet_service.cc:1467] Processing CreateTablet for tablet d254a1ad11914371a77003337335011b (DEFAULT_TABLE table=rebalance_test_table_1 [id=672006a2715b45439494a055f37d9484]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:01.102627 11009 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d254a1ad11914371a77003337335011b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:01.106006 11011 tablet_service.cc:1467] Processing CreateTablet for tablet 1b42bec0240f4547b2acdd6c8269d0eb (DEFAULT_TABLE table=rebalance_test_table_1 [id=672006a2715b45439494a055f37d9484]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:01.106355 11011 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1b42bec0240f4547b2acdd6c8269d0eb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:01.113240 11168 tablet_bootstrap.cc:492] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:01.115701 11169 tablet_bootstrap.cc:492] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:01.116079 11168 tablet_bootstrap.cc:654] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:01.117460 11169 tablet_bootstrap.cc:654] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:01.118423 11168 tablet_bootstrap.cc:492] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:01.118602 11168 ts_tablet_manager.cc:1367] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.004s
I20240502 14:17:01.119515 11169 tablet_bootstrap.cc:492] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:01.119414 11168 raft_consensus.cc:348] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.119611 11168 raft_consensus.cc:374] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:01.119653 11169 ts_tablet_manager.cc:1367] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:17:01.119714 11168 raft_consensus.cc:729] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:01.119987 11168 consensus_queue.cc:260] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.120440 11169 raft_consensus.cc:348] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.120572 11168 ts_tablet_manager.cc:1398] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:01.120648 11169 raft_consensus.cc:374] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:01.120744 11169 raft_consensus.cc:729] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:01.120939 11168 tablet_bootstrap.cc:492] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:01.120965 11169 consensus_queue.cc:260] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.121568 11169 ts_tablet_manager.cc:1398] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:01.121836 11169 tablet_bootstrap.cc:492] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:01.123378 11169 tablet_bootstrap.cc:654] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:01.125310 11169 tablet_bootstrap.cc:492] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:01.125488 11169 ts_tablet_manager.cc:1367] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:17:01.126245 11169 raft_consensus.cc:348] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.126468 11169 raft_consensus.cc:374] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:01.126559 11169 raft_consensus.cc:729] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:01.126771 11169 consensus_queue.cc:260] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.126998 11168 tablet_bootstrap.cc:654] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:01.127300 11169 ts_tablet_manager.cc:1398] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:01.127579 11169 tablet_bootstrap.cc:492] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:01.129156 11169 tablet_bootstrap.cc:654] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:01.131006 11169 tablet_bootstrap.cc:492] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:01.131063 11168 tablet_bootstrap.cc:492] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:01.131161 11169 ts_tablet_manager.cc:1367] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:17:01.131206 11168 ts_tablet_manager.cc:1367] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.003s
I20240502 14:17:01.131896 11169 raft_consensus.cc:348] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:01.131929 11168 raft_consensus.cc:348] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:01.132123 11169 raft_consensus.cc:374] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:01.132148 11168 raft_consensus.cc:374] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:01.132220 11168 raft_consensus.cc:729] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:01.132230 11169 raft_consensus.cc:729] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:01.132463 11169 consensus_queue.cc:260] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:01.132483 11168 consensus_queue.cc:260] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:01.133078 11168 ts_tablet_manager.cc:1398] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:01.133374 11168 tablet_bootstrap.cc:492] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:01.133489 11169 ts_tablet_manager.cc:1398] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.002s user 0.001s sys 0.002s
I20240502 14:17:01.135210 11168 tablet_bootstrap.cc:654] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:01.137128 11168 tablet_bootstrap.cc:492] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:01.137259 11168 ts_tablet_manager.cc:1367] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:17:01.137843 11168 raft_consensus.cc:348] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.137984 11168 raft_consensus.cc:374] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:01.138082 11168 raft_consensus.cc:729] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:01.138307 11168 consensus_queue.cc:260] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.138785 11168 ts_tablet_manager.cc:1398] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 14:17:01.240020 11113 raft_consensus.cc:482] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:01.240203 11113 raft_consensus.cc:504] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.240849 11113 leader_election.cc:288] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:01.241490 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d254a1ad11914371a77003337335011b" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b" is_pre_election: true
I20240502 14:17:01.241730 11031 raft_consensus.cc:2454] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 0.
I20240502 14:17:01.242203 10572 leader_election.cc:302] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:01.242492 11113 raft_consensus.cc:2781] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:01.242610 11113 raft_consensus.cc:482] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:01.242709 11113 raft_consensus.cc:3037] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:01.244666 11113 raft_consensus.cc:504] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.245107 11113 leader_election.cc:288] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Requested vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:01.245668 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d254a1ad11914371a77003337335011b" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b"
I20240502 14:17:01.245836 11031 raft_consensus.cc:3037] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:01.247773 11031 raft_consensus.cc:2454] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 1.
I20240502 14:17:01.248149 10572 leader_election.cc:302] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:01.248438 11113 raft_consensus.cc:2781] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:01.248565 11113 raft_consensus.cc:686] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Becoming Leader. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Running, Role: LEADER
I20240502 14:17:01.248805 11113 consensus_queue.cc:237] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.251340 10495 catalog_manager.cc:5484] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a reported cstate change: term changed from 0 to 1, leader changed from <none> to ced0bd81140c45c19192ed2611f0d58a (127.27.220.129). New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:01.296818 11113 raft_consensus.cc:482] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:01.297091 11113 raft_consensus.cc:504] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:01.297708 11113 leader_election.cc:288] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:01.298385 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b42bec0240f4547b2acdd6c8269d0eb" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b" is_pre_election: true
I20240502 14:17:01.298640 11031 raft_consensus.cc:2454] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 0.
I20240502 14:17:01.299093 10572 leader_election.cc:302] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:01.299451 11113 raft_consensus.cc:2781] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:01.299544 11113 raft_consensus.cc:482] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:01.299659 11113 raft_consensus.cc:3037] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:01.301785 11113 raft_consensus.cc:504] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:01.302256 11113 leader_election.cc:288] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Requested vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:01.302803 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b42bec0240f4547b2acdd6c8269d0eb" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b"
I20240502 14:17:01.303006 11031 raft_consensus.cc:3037] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:01.304822 11031 raft_consensus.cc:2454] T 1b42bec0240f4547b2acdd6c8269d0eb P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 1.
I20240502 14:17:01.305336 10572 leader_election.cc:302] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:01.305624 11113 raft_consensus.cc:2781] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:01.305790 11113 raft_consensus.cc:686] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Becoming Leader. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Running, Role: LEADER
I20240502 14:17:01.306126 11113 consensus_queue.cc:237] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:01.308612 10495 catalog_manager.cc:5484] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a reported cstate change: term changed from 0 to 1, leader changed from <none> to ced0bd81140c45c19192ed2611f0d58a (127.27.220.129). New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:01.388814 11119 raft_consensus.cc:482] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:01.389041 11119 raft_consensus.cc:504] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.389701 11119 leader_election.cc:288] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:01.390357 10637 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8980b56d2b04a2682f9e8c1b2c5e875" candidate_uuid: "76827763179e465a92298cea85e53f2b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced0bd81140c45c19192ed2611f0d58a" is_pre_election: true
I20240502 14:17:01.390592 10637 raft_consensus.cc:2454] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 76827763179e465a92298cea85e53f2b in term 0.
I20240502 14:17:01.391018 10966 leader_election.cc:302] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:01.391320 11119 raft_consensus.cc:2781] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:01.391429 11119 raft_consensus.cc:482] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:01.391537 11119 raft_consensus.cc:3037] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:01.393626 11119 raft_consensus.cc:504] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.394026 11119 leader_election.cc:288] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 election: Requested vote from peers ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:01.394517 10637 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e8980b56d2b04a2682f9e8c1b2c5e875" candidate_uuid: "76827763179e465a92298cea85e53f2b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced0bd81140c45c19192ed2611f0d58a"
I20240502 14:17:01.394690 10637 raft_consensus.cc:3037] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:01.396432 10637 raft_consensus.cc:2454] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 76827763179e465a92298cea85e53f2b in term 1.
I20240502 14:17:01.396823 10966 leader_election.cc:302] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:01.397050 11119 raft_consensus.cc:2781] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:01.397203 11119 raft_consensus.cc:686] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Becoming Leader. State: Replica: 76827763179e465a92298cea85e53f2b, State: Running, Role: LEADER
I20240502 14:17:01.397428 11119 consensus_queue.cc:237] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:01.399889 10495 catalog_manager.cc:5484] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b reported cstate change: term changed from 0 to 1, leader changed from <none> to 76827763179e465a92298cea85e53f2b (127.27.220.132). New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:01.716919 11113 consensus_queue.cc:1035] T 1b42bec0240f4547b2acdd6c8269d0eb P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:01.742866 11179 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:01.743208 11179 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:01.750661 11179 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 14:17:01.788784 11102 consensus_queue.cc:1035] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:01.878304 10637 raft_consensus.cc:1261] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:17:01.879031 11119 consensus_queue.cc:1035] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:01.955044 11182 mvcc.cc:205] Tried to move back new op lower bound from 7023244992007626752 to 7023244990045360128. Current Snapshot: MvccSnapshot[applied={T|T < 7023244992007626752}]
INSERT report
rows total: 2000
time total: 197.943 ms
time per row: 0.0989713 ms
I20240502 14:17:02.095479 10490 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:55026:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:02.096252 10490 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_2 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240502 14:17:02.104424 10616 tablet_service.cc:1467] Processing CreateTablet for tablet 8cb809519b7a4e77832ffaa3b2a5b51c (DEFAULT_TABLE table=rebalance_test_table_2 [id=b8142afa17ef46499325f2072b968bdd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:02.104547 11011 tablet_service.cc:1467] Processing CreateTablet for tablet 8cb809519b7a4e77832ffaa3b2a5b51c (DEFAULT_TABLE table=rebalance_test_table_2 [id=b8142afa17ef46499325f2072b968bdd]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:02.104898 10616 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8cb809519b7a4e77832ffaa3b2a5b51c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:02.105037 11011 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8cb809519b7a4e77832ffaa3b2a5b51c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:02.105122 11009 tablet_service.cc:1467] Processing CreateTablet for tablet cd799e90362e44f5905478506f9d2774 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b8142afa17ef46499325f2072b968bdd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:02.105204 10615 tablet_service.cc:1467] Processing CreateTablet for tablet cd799e90362e44f5905478506f9d2774 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b8142afa17ef46499325f2072b968bdd]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:02.105465 11009 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd799e90362e44f5905478506f9d2774. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:02.105559 10615 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd799e90362e44f5905478506f9d2774. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:02.105931 11010 tablet_service.cc:1467] Processing CreateTablet for tablet f5830021cf7e4634995dc4782c2130e2 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b8142afa17ef46499325f2072b968bdd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:02.112454 11233 tablet_bootstrap.cc:492] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:02.112713 11232 tablet_bootstrap.cc:492] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:02.113188 10615 tablet_service.cc:1467] Processing CreateTablet for tablet f5830021cf7e4634995dc4782c2130e2 (DEFAULT_TABLE table=rebalance_test_table_2 [id=b8142afa17ef46499325f2072b968bdd]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:02.113617 10615 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5830021cf7e4634995dc4782c2130e2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:02.114233 11010 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5830021cf7e4634995dc4782c2130e2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:02.114555 11233 tablet_bootstrap.cc:654] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:02.114879 11232 tablet_bootstrap.cc:654] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:02.122839 11233 tablet_bootstrap.cc:492] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:02.122994 11233 ts_tablet_manager.cc:1367] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.009s
I20240502 14:17:02.123302 11232 tablet_bootstrap.cc:492] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:02.123486 11232 ts_tablet_manager.cc:1367] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.010s
I20240502 14:17:02.123696 11233 raft_consensus.cc:348] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.123864 11233 raft_consensus.cc:374] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:02.124001 11233 raft_consensus.cc:729] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:02.124205 11233 consensus_queue.cc:260] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.124302 11232 raft_consensus.cc:348] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.124518 11232 raft_consensus.cc:374] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:02.124609 11232 raft_consensus.cc:729] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:02.124881 11233 ts_tablet_manager.cc:1398] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 14:17:02.124914 11232 consensus_queue.cc:260] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.125113 11233 tablet_bootstrap.cc:492] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:02.126147 11232 ts_tablet_manager.cc:1398] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240502 14:17:02.126483 11233 tablet_bootstrap.cc:654] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:02.126489 11232 tablet_bootstrap.cc:492] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:02.128047 11232 tablet_bootstrap.cc:654] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:02.130041 11233 tablet_bootstrap.cc:492] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:02.130261 11233 ts_tablet_manager.cc:1367] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
I20240502 14:17:02.130959 11233 raft_consensus.cc:348] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.131181 11232 tablet_bootstrap.cc:492] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:02.131242 11233 raft_consensus.cc:374] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:02.131366 11233 raft_consensus.cc:729] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:02.131413 11232 ts_tablet_manager.cc:1367] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.004s
I20240502 14:17:02.131657 11233 consensus_queue.cc:260] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.132256 11233 ts_tablet_manager.cc:1398] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:02.132190 11232 raft_consensus.cc:348] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.132400 11232 raft_consensus.cc:374] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:02.132547 11233 tablet_bootstrap.cc:492] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b: Bootstrap starting.
I20240502 14:17:02.132529 11232 raft_consensus.cc:729] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:02.132777 11232 consensus_queue.cc:260] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.133384 11232 ts_tablet_manager.cc:1398] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:02.133713 11232 tablet_bootstrap.cc:492] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: Bootstrap starting.
I20240502 14:17:02.134117 11233 tablet_bootstrap.cc:654] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:02.135898 11232 tablet_bootstrap.cc:654] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:02.136284 11233 tablet_bootstrap.cc:492] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b: No bootstrap required, opened a new log
I20240502 14:17:02.136523 11233 ts_tablet_manager.cc:1367] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:17:02.137465 11233 raft_consensus.cc:348] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:02.137710 11233 raft_consensus.cc:374] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:02.137849 11233 raft_consensus.cc:729] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 76827763179e465a92298cea85e53f2b, State: Initialized, Role: FOLLOWER
I20240502 14:17:02.138075 11233 consensus_queue.cc:260] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:02.138494 11233 ts_tablet_manager.cc:1398] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:17:02.138522 11232 tablet_bootstrap.cc:492] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: No bootstrap required, opened a new log
I20240502 14:17:02.138659 11232 ts_tablet_manager.cc:1367] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 14:17:02.139612 11232 raft_consensus.cc:348] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:02.139824 11232 raft_consensus.cc:374] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:02.139940 11232 raft_consensus.cc:729] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Initialized, Role: FOLLOWER
I20240502 14:17:02.140189 11232 consensus_queue.cc:260] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:02.140640 11232 ts_tablet_manager.cc:1398] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:02.230759 11225 raft_consensus.cc:482] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:02.230931 11225 raft_consensus.cc:504] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:02.231470 11225 leader_election.cc:288] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:02.232092 10636 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5830021cf7e4634995dc4782c2130e2" candidate_uuid: "76827763179e465a92298cea85e53f2b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced0bd81140c45c19192ed2611f0d58a" is_pre_election: true
I20240502 14:17:02.232316 10636 raft_consensus.cc:2454] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 76827763179e465a92298cea85e53f2b in term 0.
I20240502 14:17:02.232724 10966 leader_election.cc:302] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:02.232998 11225 raft_consensus.cc:2781] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:02.233094 11225 raft_consensus.cc:482] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:02.233160 11225 raft_consensus.cc:3037] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:02.235025 11225 raft_consensus.cc:504] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:02.235423 11225 leader_election.cc:288] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 election: Requested vote from peers ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:02.235860 10636 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5830021cf7e4634995dc4782c2130e2" candidate_uuid: "76827763179e465a92298cea85e53f2b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ced0bd81140c45c19192ed2611f0d58a"
I20240502 14:17:02.236016 10636 raft_consensus.cc:3037] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:02.237896 10636 raft_consensus.cc:2454] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 76827763179e465a92298cea85e53f2b in term 1.
I20240502 14:17:02.238260 10966 leader_election.cc:302] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:02.238524 11225 raft_consensus.cc:2781] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:02.238631 11225 raft_consensus.cc:686] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Becoming Leader. State: Replica: 76827763179e465a92298cea85e53f2b, State: Running, Role: LEADER
I20240502 14:17:02.238858 11225 consensus_queue.cc:237] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } }
I20240502 14:17:02.241233 10492 catalog_manager.cc:5484] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b reported cstate change: term changed from 0 to 1, leader changed from <none> to 76827763179e465a92298cea85e53f2b (127.27.220.132). New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:02.290746 11217 raft_consensus.cc:482] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:02.290956 11217 raft_consensus.cc:504] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.291548 11217 leader_election.cc:288] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:02.292186 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8cb809519b7a4e77832ffaa3b2a5b51c" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b" is_pre_election: true
I20240502 14:17:02.292387 11031 raft_consensus.cc:2454] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 0.
I20240502 14:17:02.292807 10572 leader_election.cc:302] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:02.293128 11217 raft_consensus.cc:2781] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:02.293236 11217 raft_consensus.cc:482] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:02.293329 11217 raft_consensus.cc:3037] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:02.295123 11217 raft_consensus.cc:504] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.295469 11217 leader_election.cc:288] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Requested vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:02.295948 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8cb809519b7a4e77832ffaa3b2a5b51c" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b"
I20240502 14:17:02.296090 11031 raft_consensus.cc:3037] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:02.297950 11031 raft_consensus.cc:2454] T 8cb809519b7a4e77832ffaa3b2a5b51c P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 1.
I20240502 14:17:02.298290 10572 leader_election.cc:302] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:02.298549 11217 raft_consensus.cc:2781] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:02.298673 11217 raft_consensus.cc:686] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Becoming Leader. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Running, Role: LEADER
I20240502 14:17:02.298949 11217 consensus_queue.cc:237] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.301456 10492 catalog_manager.cc:5484] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a reported cstate change: term changed from 0 to 1, leader changed from <none> to ced0bd81140c45c19192ed2611f0d58a (127.27.220.129). New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:02.334751 11217 raft_consensus.cc:482] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:02.334960 11217 raft_consensus.cc:504] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.335451 11217 leader_election.cc:288] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:02.336014 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd799e90362e44f5905478506f9d2774" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b" is_pre_election: true
I20240502 14:17:02.336266 11031 raft_consensus.cc:2454] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 0.
I20240502 14:17:02.336652 10572 leader_election.cc:302] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:02.336912 11217 raft_consensus.cc:2781] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:02.336997 11217 raft_consensus.cc:482] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:02.337097 11217 raft_consensus.cc:3037] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:02.338908 11217 raft_consensus.cc:504] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.339319 11217 leader_election.cc:288] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Requested vote from peers 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:02.339699 11031 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd799e90362e44f5905478506f9d2774" candidate_uuid: "ced0bd81140c45c19192ed2611f0d58a" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "76827763179e465a92298cea85e53f2b"
I20240502 14:17:02.339884 11031 raft_consensus.cc:3037] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:02.341694 11031 raft_consensus.cc:2454] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ced0bd81140c45c19192ed2611f0d58a in term 1.
I20240502 14:17:02.342072 10572 leader_election.cc:302] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 76827763179e465a92298cea85e53f2b, ced0bd81140c45c19192ed2611f0d58a; no voters:
I20240502 14:17:02.342324 11217 raft_consensus.cc:2781] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:02.342504 11217 raft_consensus.cc:686] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Becoming Leader. State: Replica: ced0bd81140c45c19192ed2611f0d58a, State: Running, Role: LEADER
I20240502 14:17:02.342736 11217 consensus_queue.cc:237] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } }
I20240502 14:17:02.344985 10492 catalog_manager.cc:5484] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a reported cstate change: term changed from 0 to 1, leader changed from <none> to ced0bd81140c45c19192ed2611f0d58a (127.27.220.129). New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: UNKNOWN } } }
W20240502 14:17:02.690274 11243 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:02.690593 11243 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:02.697957 11243 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:17:02.725114 11217 consensus_queue.cc:1035] T 8cb809519b7a4e77832ffaa3b2a5b51c P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
I20240502 14:17:02.813335 11225 consensus_queue.cc:1035] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:02.860050 11031 raft_consensus.cc:1261] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 14:17:02.861804 11102 consensus_queue.cc:1035] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:02.936393 11279 mvcc.cc:205] Tried to move back new op lower bound from 7023244996027154432 to 7023244993917296640. Current Snapshot: MvccSnapshot[applied={T|T < 7023244996027154432}]
INSERT report
rows total: 2000
time total: 220.043 ms
time per row: 0.110021 ms
I20240502 14:17:03.070093 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.130:33583
--local_ip_for_outbound_sockets=127.27.220.130
--tserver_master_addrs=127.27.220.190:44381
--webserver_port=43281
--webserver_interface=127.27.220.130
--builtin_ntp_servers=127.27.220.148:34559
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:17:03.403833 11295 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:03.404131 11295 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:03.404222 11295 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:03.411578 11295 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:03.411701 11295 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:03.411813 11295 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.130
I20240502 14:17:03.420981 11295 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:34559
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.130:33583
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.27.220.130
--webserver_port=43281
--tserver_master_addrs=127.27.220.190:44381
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.130
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:03.422291 11295 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:03.424011 11295 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:03.436143 11303 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:03.436968 11300 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:03.437167 11301 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:03.437641 11295 server_base.cc:1034] running on GCE node
I20240502 14:17:03.438213 11295 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:03.439117 11295 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:03.440320 11295 hybrid_clock.cc:648] HybridClock initialized: now 1714659423440264 us; error 53 us; skew 500 ppm
I20240502 14:17:03.440650 11295 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:03.442991 11295 webserver.cc:460] Webserver started at http://127.27.220.130:43281/ using document root <none> and password file <none>
I20240502 14:17:03.443774 11295 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:03.443915 11295 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:03.448971 11295 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.001s sys 0.001s
I20240502 14:17:03.451699 11309 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:03.453150 11295 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:03.453325 11295 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "3be4f0789bca4f0483a4d3df81a8b0c3"
format_stamp: "Formatted at 2024-05-02 14:16:54 on dist-test-slave-j72s"
I20240502 14:17:03.454353 11295 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:03.473206 11295 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:03.474090 11295 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:03.474400 11295 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:03.475277 11295 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:03.476699 11295 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:03.476778 11295 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:03.476912 11295 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:03.476967 11295 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:03.507789 11295 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.130:33583
I20240502 14:17:03.507828 11422 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.130:33583 every 8 connection(s)
I20240502 14:17:03.509179 11295 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 14:17:03.519433 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 11295
I20240502 14:17:03.520063 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:38943
--local_ip_for_outbound_sockets=127.27.220.131
--tserver_master_addrs=127.27.220.190:44381
--webserver_port=33383
--webserver_interface=127.27.220.131
--builtin_ntp_servers=127.27.220.148:34559
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:03.521530 11423 heartbeater.cc:347] Connected to a master server at 127.27.220.190:44381
I20240502 14:17:03.521806 11423 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:03.522568 11423 heartbeater.cc:511] Master 127.27.220.190:44381 requested a full tablet report, sending...
I20240502 14:17:03.524175 10489 ts_manager.cc:194] Re-registered known tserver with Master: 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130:33583)
I20240502 14:17:03.525017 10489 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.130:42227
W20240502 14:17:03.845086 11427 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:03.845346 11427 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:03.845438 11427 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:03.852475 11427 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:03.852591 11427 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:03.852708 11427 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:17:03.861495 11427 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:34559
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:38943
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=33383
--tserver_master_addrs=127.27.220.190:44381
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:03.862754 11427 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:03.864396 11427 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:03.876073 11432 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:03.876571 11433 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:03.877137 11435 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:03.877543 11427 server_base.cc:1034] running on GCE node
I20240502 14:17:03.878072 11427 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:03.878937 11427 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:03.880111 11427 hybrid_clock.cc:648] HybridClock initialized: now 1714659423880063 us; error 43 us; skew 500 ppm
I20240502 14:17:03.880406 11427 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:03.882638 11427 webserver.cc:460] Webserver started at http://127.27.220.131:33383/ using document root <none> and password file <none>
I20240502 14:17:03.883322 11427 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:03.883442 11427 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:03.888446 11427 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 14:17:03.890970 11441 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:03.892232 11427 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240502 14:17:03.892385 11427 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "4224d6603b434d6cb7cf3b58a86a041f"
format_stamp: "Formatted at 2024-05-02 14:16:55 on dist-test-slave-j72s"
I20240502 14:17:03.893190 11427 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:03.938141 11427 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:03.939013 11427 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:03.939333 11427 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:03.940290 11427 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:03.942198 11427 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:03.942281 11427 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:03.942425 11427 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:03.942515 11427 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:03.991375 11427 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:38943
I20240502 14:17:03.991459 11553 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:38943 every 8 connection(s)
I20240502 14:17:03.992929 11427 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:17:03.999715 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 11427
I20240502 14:17:04.014710 11554 heartbeater.cc:347] Connected to a master server at 127.27.220.190:44381
I20240502 14:17:04.015038 11554 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:04.015883 11554 heartbeater.cc:511] Master 127.27.220.190:44381 requested a full tablet report, sending...
I20240502 14:17:04.017458 10489 ts_manager.cc:194] Re-registered known tserver with Master: 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131:38943)
I20240502 14:17:04.018265 10489 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:47297
I20240502 14:17:04.461687 11489 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:04.462201 11358 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:04.472364 10615 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:17:04.480214 11010 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:04.527895 11423 heartbeater.cc:503] Master 127.27.220.190:44381 was elected leader, sending a full tablet report...
I20240502 14:17:04.637791 11358 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:04.638563 11489 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:04.641276 11010 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:04.658985 10615 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:17:04.821583 11358 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:04.823468 11489 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:04.829131 11010 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:04.838686 10615 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:17:04.970054 11358 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:04.986174 11489 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:04.990124 10615 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:17:04.992380 11010 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:05.020637 11554 heartbeater.cc:503] Master 127.27.220.190:44381 was elected leader, sending a full tablet report...
I20240502 14:17:05.065778 10636 consensus_queue.cc:237] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.068504 11031 raft_consensus.cc:1261] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:05.069061 11292 consensus_queue.cc:1035] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:05.072206 11693 raft_consensus.cc:2932] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } } }
I20240502 14:17:05.072952 11031 raft_consensus.cc:2932] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } } }
W20240502 14:17:05.075824 10570 consensus_peers.cc:485] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a -> Peer 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130:33583): Couldn't send request to peer 3be4f0789bca4f0483a4d3df81a8b0c3. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd799e90362e44f5905478506f9d2774. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:05.079056 10492 catalog_manager.cc:5484] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a reported cstate change: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:05.092078 11031 consensus_queue.cc:237] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
I20240502 14:17:05.094362 10636 raft_consensus.cc:1261] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:05.094903 11275 consensus_queue.cc:1035] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:05.097684 11706 raft_consensus.cc:2932] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } } }
I20240502 14:17:05.098438 10636 raft_consensus.cc:2932] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } } }
W20240502 14:17:05.103747 10967 consensus_peers.cc:485] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b -> Peer 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131:38943): Couldn't send request to peer 4224d6603b434d6cb7cf3b58a86a041f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e8980b56d2b04a2682f9e8c1b2c5e875. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:05.104444 10492 catalog_manager.cc:5484] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b reported cstate change: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:05.117385 10636 consensus_queue.cc:237] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.119298 11031 raft_consensus.cc:1261] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 14:17:05.119494 10570 consensus_peers.cc:485] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a -> Peer 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130:33583): Couldn't send request to peer 3be4f0789bca4f0483a4d3df81a8b0c3. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d254a1ad11914371a77003337335011b. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:05.119802 11292 consensus_queue.cc:1035] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:05.122126 11693 raft_consensus.cc:2932] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } } }
I20240502 14:17:05.122777 11031 raft_consensus.cc:2932] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } } }
I20240502 14:17:05.127146 10489 catalog_manager.cc:5484] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a reported cstate change: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:05.143976 11031 consensus_queue.cc:237] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
W20240502 14:17:05.146288 10967 consensus_peers.cc:485] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b -> Peer 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131:38943): Couldn't send request to peer 4224d6603b434d6cb7cf3b58a86a041f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f5830021cf7e4634995dc4782c2130e2. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:05.146286 10636 raft_consensus.cc:1261] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:05.146802 11275 consensus_queue.cc:1035] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:05.149196 11706 raft_consensus.cc:2932] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } } }
I20240502 14:17:05.149832 10636 raft_consensus.cc:2932] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } } }
I20240502 14:17:05.153810 10492 catalog_manager.cc:5484] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b reported cstate change: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:05.166033 10636 consensus_queue.cc:237] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
I20240502 14:17:05.168196 11031 raft_consensus.cc:1261] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:05.168843 11292 consensus_queue.cc:1035] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:05.171866 11693 raft_consensus.cc:2932] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } } }
I20240502 14:17:05.172547 11031 raft_consensus.cc:2932] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } } }
W20240502 14:17:05.174415 10573 consensus_peers.cc:485] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a -> Peer 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131:38943): Couldn't send request to peer 4224d6603b434d6cb7cf3b58a86a041f. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ab98a42082f34b839684b25d6386898b. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:05.177666 10492 catalog_manager.cc:5484] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b reported cstate change: config changed from index -1 to 4, NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) added. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } } }
I20240502 14:17:05.189462 11031 consensus_queue.cc:237] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.191721 10636 raft_consensus.cc:1261] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:05.192414 11723 consensus_queue.cc:1035] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:05.195622 11275 raft_consensus.cc:2932] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } } }
I20240502 14:17:05.196367 10636 raft_consensus.cc:2932] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } } }
W20240502 14:17:05.198539 10964 consensus_peers.cc:485] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b -> Peer 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130:33583): Couldn't send request to peer 3be4f0789bca4f0483a4d3df81a8b0c3. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e4876c504a97477cb27fb7f6c4ae12b6. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:05.202221 10489 catalog_manager.cc:5484] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b reported cstate change: config changed from index -1 to 4, NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) added. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:05.534066 11799 ts_tablet_manager.cc:902] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: Initiating tablet copy from peer 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:05.535964 11799 tablet_copy_client.cc:308] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:35055
I20240502 14:17:05.542903 11051 tablet_copy_service.cc:140] P 76827763179e465a92298cea85e53f2b: Received BeginTabletCopySession request for tablet f5830021cf7e4634995dc4782c2130e2 from peer 4224d6603b434d6cb7cf3b58a86a041f ({username='slave'} at 127.27.220.131:45903)
I20240502 14:17:05.543154 11051 tablet_copy_service.cc:161] P 76827763179e465a92298cea85e53f2b: Beginning new tablet copy session on tablet f5830021cf7e4634995dc4782c2130e2 from peer 4224d6603b434d6cb7cf3b58a86a041f at {username='slave'} at 127.27.220.131:45903: session id = 4224d6603b434d6cb7cf3b58a86a041f-f5830021cf7e4634995dc4782c2130e2
I20240502 14:17:05.545917 11051 tablet_copy_source_session.cc:205] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:05.551255 11799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5830021cf7e4634995dc4782c2130e2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:05.566390 11799 tablet_copy_client.cc:783] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:05.566741 11799 tablet_copy_client.cc:647] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:05.569540 11799 tablet_copy_client.cc:523] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:05.572628 11799 tablet_bootstrap.cc:492] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap starting.
I20240502 14:17:05.581753 11799 log.cc:824] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:05.611706 11799 tablet_bootstrap.cc:492] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:05.612232 11799 tablet_bootstrap.cc:492] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap complete.
I20240502 14:17:05.612679 11799 ts_tablet_manager.cc:1367] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: Time spent bootstrapping tablet: real 0.040s user 0.032s sys 0.008s
I20240502 14:17:05.617676 11806 ts_tablet_manager.cc:902] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: Initiating tablet copy from peer ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:05.617664 11799 raft_consensus.cc:348] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
I20240502 14:17:05.618103 11799 raft_consensus.cc:729] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4224d6603b434d6cb7cf3b58a86a041f, State: Initialized, Role: LEARNER
I20240502 14:17:05.618721 11799 consensus_queue.cc:260] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
I20240502 14:17:05.619522 11806 tablet_copy_client.cc:308] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:37689
I20240502 14:17:05.620563 11799 ts_tablet_manager.cc:1398] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f: Time spent starting tablet: real 0.008s user 0.010s sys 0.000s
I20240502 14:17:05.621809 11051 tablet_copy_service.cc:342] P 76827763179e465a92298cea85e53f2b: Request end of tablet copy session 4224d6603b434d6cb7cf3b58a86a041f-f5830021cf7e4634995dc4782c2130e2 received from {username='slave'} at 127.27.220.131:45903
I20240502 14:17:05.621989 11051 tablet_copy_service.cc:433] P 76827763179e465a92298cea85e53f2b: ending tablet copy session 4224d6603b434d6cb7cf3b58a86a041f-f5830021cf7e4634995dc4782c2130e2 on tablet f5830021cf7e4634995dc4782c2130e2 with peer 4224d6603b434d6cb7cf3b58a86a041f
I20240502 14:17:05.627153 10657 tablet_copy_service.cc:140] P ced0bd81140c45c19192ed2611f0d58a: Received BeginTabletCopySession request for tablet ab98a42082f34b839684b25d6386898b from peer 4224d6603b434d6cb7cf3b58a86a041f ({username='slave'} at 127.27.220.131:60953)
I20240502 14:17:05.627391 10657 tablet_copy_service.cc:161] P ced0bd81140c45c19192ed2611f0d58a: Beginning new tablet copy session on tablet ab98a42082f34b839684b25d6386898b from peer 4224d6603b434d6cb7cf3b58a86a041f at {username='slave'} at 127.27.220.131:60953: session id = 4224d6603b434d6cb7cf3b58a86a041f-ab98a42082f34b839684b25d6386898b
I20240502 14:17:05.629953 10657 tablet_copy_source_session.cc:205] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:05.631417 11806 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ab98a42082f34b839684b25d6386898b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:05.635520 11812 ts_tablet_manager.cc:902] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: Initiating tablet copy from peer ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:05.636152 11806 tablet_copy_client.cc:783] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:05.636444 11806 tablet_copy_client.cc:647] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:05.638288 11812 tablet_copy_client.cc:308] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:37689
I20240502 14:17:05.638767 11806 tablet_copy_client.cc:523] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:05.641491 11806 tablet_bootstrap.cc:492] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap starting.
I20240502 14:17:05.644623 10657 tablet_copy_service.cc:140] P ced0bd81140c45c19192ed2611f0d58a: Received BeginTabletCopySession request for tablet d254a1ad11914371a77003337335011b from peer 3be4f0789bca4f0483a4d3df81a8b0c3 ({username='slave'} at 127.27.220.130:41915)
I20240502 14:17:05.644831 10657 tablet_copy_service.cc:161] P ced0bd81140c45c19192ed2611f0d58a: Beginning new tablet copy session on tablet d254a1ad11914371a77003337335011b from peer 3be4f0789bca4f0483a4d3df81a8b0c3 at {username='slave'} at 127.27.220.130:41915: session id = 3be4f0789bca4f0483a4d3df81a8b0c3-d254a1ad11914371a77003337335011b
I20240502 14:17:05.647027 10657 tablet_copy_source_session.cc:205] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:05.651371 11812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d254a1ad11914371a77003337335011b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:05.661859 11812 tablet_copy_client.cc:783] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:05.662225 11812 tablet_copy_client.cc:647] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:05.664340 11812 tablet_copy_client.cc:523] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:05.667449 11812 tablet_bootstrap.cc:492] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap starting.
I20240502 14:17:05.671658 11819 ts_tablet_manager.cc:902] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: Initiating tablet copy from peer ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689)
I20240502 14:17:05.673465 11806 tablet_bootstrap.cc:492] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:05.674165 11806 tablet_bootstrap.cc:492] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap complete.
I20240502 14:17:05.674585 11806 ts_tablet_manager.cc:1367] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: Time spent bootstrapping tablet: real 0.033s user 0.027s sys 0.009s
I20240502 14:17:05.674592 11819 tablet_copy_client.cc:308] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:37689
I20240502 14:17:05.675472 10657 tablet_copy_service.cc:140] P ced0bd81140c45c19192ed2611f0d58a: Received BeginTabletCopySession request for tablet cd799e90362e44f5905478506f9d2774 from peer 3be4f0789bca4f0483a4d3df81a8b0c3 ({username='slave'} at 127.27.220.130:41915)
I20240502 14:17:05.675521 11806 raft_consensus.cc:348] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
I20240502 14:17:05.675674 10657 tablet_copy_service.cc:161] P ced0bd81140c45c19192ed2611f0d58a: Beginning new tablet copy session on tablet cd799e90362e44f5905478506f9d2774 from peer 3be4f0789bca4f0483a4d3df81a8b0c3 at {username='slave'} at 127.27.220.130:41915: session id = 3be4f0789bca4f0483a4d3df81a8b0c3-cd799e90362e44f5905478506f9d2774
I20240502 14:17:05.675777 11806 raft_consensus.cc:729] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4224d6603b434d6cb7cf3b58a86a041f, State: Initialized, Role: LEARNER
I20240502 14:17:05.675998 11806 consensus_queue.cc:260] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
I20240502 14:17:05.676481 11806 ts_tablet_manager.cc:1398] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:05.677608 10656 tablet_copy_service.cc:342] P ced0bd81140c45c19192ed2611f0d58a: Request end of tablet copy session 4224d6603b434d6cb7cf3b58a86a041f-ab98a42082f34b839684b25d6386898b received from {username='slave'} at 127.27.220.131:60953
I20240502 14:17:05.677783 10656 tablet_copy_service.cc:433] P ced0bd81140c45c19192ed2611f0d58a: ending tablet copy session 4224d6603b434d6cb7cf3b58a86a041f-ab98a42082f34b839684b25d6386898b on tablet ab98a42082f34b839684b25d6386898b with peer 4224d6603b434d6cb7cf3b58a86a041f
I20240502 14:17:05.678803 10657 tablet_copy_source_session.cc:205] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:05.680788 11819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd799e90362e44f5905478506f9d2774. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:05.681272 11812 log.cc:824] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:05.684329 11825 ts_tablet_manager.cc:902] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: Initiating tablet copy from peer 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:05.687629 11819 tablet_copy_client.cc:783] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:05.687929 11819 tablet_copy_client.cc:647] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:05.690229 11819 tablet_copy_client.cc:523] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:05.692752 11825 tablet_copy_client.cc:308] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:35055
I20240502 14:17:05.694566 11819 tablet_bootstrap.cc:492] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap starting.
I20240502 14:17:05.700518 11051 tablet_copy_service.cc:140] P 76827763179e465a92298cea85e53f2b: Received BeginTabletCopySession request for tablet e4876c504a97477cb27fb7f6c4ae12b6 from peer 3be4f0789bca4f0483a4d3df81a8b0c3 ({username='slave'} at 127.27.220.130:33657)
I20240502 14:17:05.700706 11051 tablet_copy_service.cc:161] P 76827763179e465a92298cea85e53f2b: Beginning new tablet copy session on tablet e4876c504a97477cb27fb7f6c4ae12b6 from peer 3be4f0789bca4f0483a4d3df81a8b0c3 at {username='slave'} at 127.27.220.130:33657: session id = 3be4f0789bca4f0483a4d3df81a8b0c3-e4876c504a97477cb27fb7f6c4ae12b6
I20240502 14:17:05.703441 11051 tablet_copy_source_session.cc:205] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:05.704735 11825 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e4876c504a97477cb27fb7f6c4ae12b6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:05.711567 11825 tablet_copy_client.cc:783] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:05.711882 11825 tablet_copy_client.cc:647] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:05.713873 11825 tablet_copy_client.cc:523] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:05.714421 11806 ts_tablet_manager.cc:902] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: Initiating tablet copy from peer 76827763179e465a92298cea85e53f2b (127.27.220.132:35055)
I20240502 14:17:05.717590 11825 tablet_bootstrap.cc:492] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap starting.
I20240502 14:17:05.724707 11812 tablet_bootstrap.cc:492] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:05.725695 11812 tablet_bootstrap.cc:492] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap complete.
I20240502 14:17:05.726326 11812 ts_tablet_manager.cc:1367] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: Time spent bootstrapping tablet: real 0.059s user 0.046s sys 0.011s
I20240502 14:17:05.735021 11812 raft_consensus.cc:348] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.735652 11812 raft_consensus.cc:729] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3be4f0789bca4f0483a4d3df81a8b0c3, State: Initialized, Role: LEARNER
I20240502 14:17:05.736428 11812 consensus_queue.cc:260] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.744756 11812 ts_tablet_manager.cc:1398] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3: Time spent starting tablet: real 0.018s user 0.012s sys 0.005s
I20240502 14:17:05.751289 10657 tablet_copy_service.cc:342] P ced0bd81140c45c19192ed2611f0d58a: Request end of tablet copy session 3be4f0789bca4f0483a4d3df81a8b0c3-d254a1ad11914371a77003337335011b received from {username='slave'} at 127.27.220.130:41915
I20240502 14:17:05.751451 10657 tablet_copy_service.cc:433] P ced0bd81140c45c19192ed2611f0d58a: ending tablet copy session 3be4f0789bca4f0483a4d3df81a8b0c3-d254a1ad11914371a77003337335011b on tablet d254a1ad11914371a77003337335011b with peer 3be4f0789bca4f0483a4d3df81a8b0c3
I20240502 14:17:05.754449 11806 tablet_copy_client.cc:308] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.132:35055
I20240502 14:17:05.755321 11051 tablet_copy_service.cc:140] P 76827763179e465a92298cea85e53f2b: Received BeginTabletCopySession request for tablet e8980b56d2b04a2682f9e8c1b2c5e875 from peer 4224d6603b434d6cb7cf3b58a86a041f ({username='slave'} at 127.27.220.131:45903)
I20240502 14:17:05.755481 11051 tablet_copy_service.cc:161] P 76827763179e465a92298cea85e53f2b: Beginning new tablet copy session on tablet e8980b56d2b04a2682f9e8c1b2c5e875 from peer 4224d6603b434d6cb7cf3b58a86a041f at {username='slave'} at 127.27.220.131:45903: session id = 4224d6603b434d6cb7cf3b58a86a041f-e8980b56d2b04a2682f9e8c1b2c5e875
I20240502 14:17:05.756155 11819 tablet_bootstrap.cc:492] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:05.756625 11819 tablet_bootstrap.cc:492] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap complete.
I20240502 14:17:05.756923 11819 ts_tablet_manager.cc:1367] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: Time spent bootstrapping tablet: real 0.062s user 0.058s sys 0.000s
I20240502 14:17:05.757493 11051 tablet_copy_source_session.cc:205] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:05.757931 11819 raft_consensus.cc:348] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.758225 11819 raft_consensus.cc:729] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3be4f0789bca4f0483a4d3df81a8b0c3, State: Initialized, Role: LEARNER
I20240502 14:17:05.758445 11819 consensus_queue.cc:260] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.759316 11806 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e8980b56d2b04a2682f9e8c1b2c5e875. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:05.761593 11819 ts_tablet_manager.cc:1398] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 14:17:05.764528 10657 tablet_copy_service.cc:342] P ced0bd81140c45c19192ed2611f0d58a: Request end of tablet copy session 3be4f0789bca4f0483a4d3df81a8b0c3-cd799e90362e44f5905478506f9d2774 received from {username='slave'} at 127.27.220.130:41915
I20240502 14:17:05.764659 10657 tablet_copy_service.cc:433] P ced0bd81140c45c19192ed2611f0d58a: ending tablet copy session 3be4f0789bca4f0483a4d3df81a8b0c3-cd799e90362e44f5905478506f9d2774 on tablet cd799e90362e44f5905478506f9d2774 with peer 3be4f0789bca4f0483a4d3df81a8b0c3
I20240502 14:17:05.767776 11806 tablet_copy_client.cc:783] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:05.768643 11806 tablet_copy_client.cc:647] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:05.770357 11806 tablet_copy_client.cc:523] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:05.773566 11806 tablet_bootstrap.cc:492] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap starting.
I20240502 14:17:05.777453 11825 tablet_bootstrap.cc:492] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:05.777848 11825 tablet_bootstrap.cc:492] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: Bootstrap complete.
I20240502 14:17:05.778081 11825 ts_tablet_manager.cc:1367] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: Time spent bootstrapping tablet: real 0.061s user 0.033s sys 0.023s
I20240502 14:17:05.778658 11825 raft_consensus.cc:348] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.778849 11825 raft_consensus.cc:729] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3be4f0789bca4f0483a4d3df81a8b0c3, State: Initialized, Role: LEARNER
I20240502 14:17:05.779037 11825 consensus_queue.cc:260] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: NON_VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: true } }
I20240502 14:17:05.779527 11825 ts_tablet_manager.cc:1398] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 14:17:05.780479 11051 tablet_copy_service.cc:342] P 76827763179e465a92298cea85e53f2b: Request end of tablet copy session 3be4f0789bca4f0483a4d3df81a8b0c3-e4876c504a97477cb27fb7f6c4ae12b6 received from {username='slave'} at 127.27.220.130:33657
I20240502 14:17:05.780620 11051 tablet_copy_service.cc:433] P 76827763179e465a92298cea85e53f2b: ending tablet copy session 3be4f0789bca4f0483a4d3df81a8b0c3-e4876c504a97477cb27fb7f6c4ae12b6 on tablet e4876c504a97477cb27fb7f6c4ae12b6 with peer 3be4f0789bca4f0483a4d3df81a8b0c3
I20240502 14:17:05.818222 11806 tablet_bootstrap.cc:492] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:05.818686 11806 tablet_bootstrap.cc:492] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: Bootstrap complete.
I20240502 14:17:05.819010 11806 ts_tablet_manager.cc:1367] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: Time spent bootstrapping tablet: real 0.046s user 0.041s sys 0.005s
I20240502 14:17:05.819929 11806 raft_consensus.cc:348] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
I20240502 14:17:05.820225 11806 raft_consensus.cc:729] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4224d6603b434d6cb7cf3b58a86a041f, State: Initialized, Role: LEARNER
I20240502 14:17:05.820494 11806 consensus_queue.cc:260] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: NON_VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: true } }
I20240502 14:17:05.821064 11806 ts_tablet_manager.cc:1398] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:17:05.821924 11051 tablet_copy_service.cc:342] P 76827763179e465a92298cea85e53f2b: Request end of tablet copy session 4224d6603b434d6cb7cf3b58a86a041f-e8980b56d2b04a2682f9e8c1b2c5e875 received from {username='slave'} at 127.27.220.131:45903
I20240502 14:17:05.822085 11051 tablet_copy_service.cc:433] P 76827763179e465a92298cea85e53f2b: ending tablet copy session 4224d6603b434d6cb7cf3b58a86a041f-e8980b56d2b04a2682f9e8c1b2c5e875 on tablet e8980b56d2b04a2682f9e8c1b2c5e875 with peer 4224d6603b434d6cb7cf3b58a86a041f
I20240502 14:17:06.065217 11509 raft_consensus.cc:1203] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:17:06.095407 11378 raft_consensus.cc:1203] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:17:06.127588 11378 raft_consensus.cc:1203] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:17:06.189976 11509 raft_consensus.cc:1203] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:17:06.214010 11378 raft_consensus.cc:1203] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:17:06.292778 11509 raft_consensus.cc:1203] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 14:17:06.585577 11890 raft_consensus.cc:1050] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a: attempting to promote NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 to VOTER
I20240502 14:17:06.586463 11890 consensus_queue.cc:237] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } }
I20240502 14:17:06.589401 11378 raft_consensus.cc:1261] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.589421 11031 raft_consensus.cc:1261] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.590095 11890 consensus_queue.cc:1035] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.590472 11886 consensus_queue.cc:1035] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.593202 11890 raft_consensus.cc:2932] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.593930 11031 raft_consensus.cc:2932] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.595525 11378 raft_consensus.cc:2932] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.598477 10492 catalog_manager.cc:5484] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a reported cstate change: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:06.607422 11884 raft_consensus.cc:1050] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b: attempting to promote NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f to VOTER
I20240502 14:17:06.608127 11884 consensus_queue.cc:237] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } }
I20240502 14:17:06.611649 10636 raft_consensus.cc:1261] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.611771 11509 raft_consensus.cc:1261] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.612478 11885 consensus_queue.cc:1035] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.612771 11879 consensus_queue.cc:1035] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.616390 11879 raft_consensus.cc:2932] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.617282 11890 raft_consensus.cc:1050] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a: attempting to promote NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 to VOTER
I20240502 14:17:06.618150 11890 consensus_queue.cc:237] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } }
I20240502 14:17:06.620887 10637 raft_consensus.cc:2932] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.621402 10636 consensus_queue.cc:237] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } }
I20240502 14:17:06.623867 11378 raft_consensus.cc:1261] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:06.624523 11886 consensus_queue.cc:1035] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:06.619222 11509 raft_consensus.cc:2932] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.627343 10490 catalog_manager.cc:5484] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b reported cstate change: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:06.627475 11886 raft_consensus.cc:2932] T cd799e90362e44f5905478506f9d2774 P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.630195 11378 raft_consensus.cc:2932] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.633436 11031 consensus_queue.cc:237] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } }
I20240502 14:17:06.635908 11509 raft_consensus.cc:1261] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:06.636629 10489 catalog_manager.cc:5484] T cd799e90362e44f5905478506f9d2774 P 3be4f0789bca4f0483a4d3df81a8b0c3 reported cstate change: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.637652 11885 consensus_queue.cc:1035] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:06.640671 10482 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd799e90362e44f5905478506f9d2774 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:06.642246 11885 raft_consensus.cc:2932] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.642958 11010 tablet_service.cc:1514] Processing DeleteTablet for tablet cd799e90362e44f5905478506f9d2774 with delete_type TABLET_DATA_TOMBSTONED (TS 76827763179e465a92298cea85e53f2b not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52404
I20240502 14:17:06.645660 10482 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f5830021cf7e4634995dc4782c2130e2 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:06.646095 11509 raft_consensus.cc:2932] T f5830021cf7e4634995dc4782c2130e2 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.646380 11031 raft_consensus.cc:1261] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.646726 10489 catalog_manager.cc:5484] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b reported cstate change: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:06.646988 11693 consensus_queue.cc:1035] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.648016 11378 raft_consensus.cc:1261] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.648909 11693 consensus_queue.cc:1035] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.650131 11693 raft_consensus.cc:2932] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.654215 11030 raft_consensus.cc:2932] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.654824 10615 tablet_service.cc:1514] Processing DeleteTablet for tablet f5830021cf7e4634995dc4782c2130e2 with delete_type TABLET_DATA_TOMBSTONED (TS ced0bd81140c45c19192ed2611f0d58a not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53314
I20240502 14:17:06.660264 11378 raft_consensus.cc:2932] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.661973 11963 tablet_replica.cc:330] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: stopping tablet replica
I20240502 14:17:06.662500 11963 raft_consensus.cc:2229] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:06.662955 11963 raft_consensus.cc:2258] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:06.665136 11963 ts_tablet_manager.cc:1875] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:06.666082 10489 catalog_manager.cc:5484] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 reported cstate change: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.670967 11886 raft_consensus.cc:1050] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a: attempting to promote NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f to VOTER
I20240502 14:17:06.671751 11886 consensus_queue.cc:237] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } }
I20240502 14:17:06.672530 11963 ts_tablet_manager.cc:1888] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:17:06.672678 11963 log.cc:1196] T f5830021cf7e4634995dc4782c2130e2 P ced0bd81140c45c19192ed2611f0d58a: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/f5830021cf7e4634995dc4782c2130e2
I20240502 14:17:06.674103 10482 catalog_manager.cc:4830] TS ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689): tablet f5830021cf7e4634995dc4782c2130e2 (table rebalance_test_table_2 [id=b8142afa17ef46499325f2072b968bdd]) successfully deleted
I20240502 14:17:06.677693 10637 consensus_queue.cc:237] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } }
I20240502 14:17:06.679062 11378 raft_consensus.cc:1261] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:06.679653 11890 consensus_queue.cc:1035] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:06.681703 11693 raft_consensus.cc:2932] T d254a1ad11914371a77003337335011b P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.682152 11378 raft_consensus.cc:2932] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.686372 10482 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d254a1ad11914371a77003337335011b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:06.690269 10490 catalog_manager.cc:5484] T d254a1ad11914371a77003337335011b P 3be4f0789bca4f0483a4d3df81a8b0c3 reported cstate change: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.694344 11009 tablet_service.cc:1514] Processing DeleteTablet for tablet d254a1ad11914371a77003337335011b with delete_type TABLET_DATA_TOMBSTONED (TS 76827763179e465a92298cea85e53f2b not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52404
I20240502 14:17:06.697746 11959 tablet_replica.cc:330] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: stopping tablet replica
I20240502 14:17:06.697755 11509 raft_consensus.cc:1261] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.698158 11959 raft_consensus.cc:2229] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:06.698558 11959 raft_consensus.cc:2258] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:06.698539 11890 consensus_queue.cc:1035] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.699896 11959 ts_tablet_manager.cc:1875] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:06.701555 11030 raft_consensus.cc:1261] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.702451 11509 raft_consensus.cc:2932] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.705008 11959 ts_tablet_manager.cc:1888] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:17:06.705124 11959 log.cc:1196] T cd799e90362e44f5905478506f9d2774 P 76827763179e465a92298cea85e53f2b: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/cd799e90362e44f5905478506f9d2774
I20240502 14:17:06.705888 11959 tablet_replica.cc:330] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: stopping tablet replica
I20240502 14:17:06.705905 10482 catalog_manager.cc:4830] TS 76827763179e465a92298cea85e53f2b (127.27.220.132:35055): tablet cd799e90362e44f5905478506f9d2774 (table rebalance_test_table_2 [id=b8142afa17ef46499325f2072b968bdd]) successfully deleted
I20240502 14:17:06.705680 11693 raft_consensus.cc:2932] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.706192 11959 raft_consensus.cc:2229] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:06.706501 11959 raft_consensus.cc:2258] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:06.707449 11959 ts_tablet_manager.cc:1875] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:06.708859 11890 consensus_queue.cc:1035] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.714313 11959 ts_tablet_manager.cc:1888] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:17:06.714511 11959 log.cc:1196] T d254a1ad11914371a77003337335011b P 76827763179e465a92298cea85e53f2b: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/d254a1ad11914371a77003337335011b
I20240502 14:17:06.715358 10482 catalog_manager.cc:4830] TS 76827763179e465a92298cea85e53f2b (127.27.220.132:35055): tablet d254a1ad11914371a77003337335011b (table rebalance_test_table_1 [id=672006a2715b45439494a055f37d9484]) successfully deleted
I20240502 14:17:06.722566 10490 catalog_manager.cc:5484] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f reported cstate change: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.726143 11030 raft_consensus.cc:2932] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.728039 11885 raft_consensus.cc:1050] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b: attempting to promote NON_VOTER 3be4f0789bca4f0483a4d3df81a8b0c3 to VOTER
I20240502 14:17:06.728782 11885 consensus_queue.cc:237] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } }
I20240502 14:17:06.733383 10637 consensus_queue.cc:237] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } }
I20240502 14:17:06.734392 11378 raft_consensus.cc:1261] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 LEARNER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.734894 11885 consensus_queue.cc:1035] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.735534 11509 raft_consensus.cc:1261] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Refusing update from remote peer ced0bd81140c45c19192ed2611f0d58a: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:06.736168 11890 consensus_queue.cc:1035] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [LEADER]: Connected to new peer: Peer: permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:06.738024 10637 raft_consensus.cc:1261] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.738538 11693 raft_consensus.cc:2932] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.738745 11885 consensus_queue.cc:1035] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.739110 11509 raft_consensus.cc:2932] T ab98a42082f34b839684b25d6386898b P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.739243 11885 raft_consensus.cc:2932] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.742189 10482 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ab98a42082f34b839684b25d6386898b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:06.742271 11378 raft_consensus.cc:2932] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.743429 10490 catalog_manager.cc:5484] T ab98a42082f34b839684b25d6386898b P ced0bd81140c45c19192ed2611f0d58a reported cstate change: config changed from index 5 to 6, VOTER 76827763179e465a92298cea85e53f2b (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "ced0bd81140c45c19192ed2611f0d58a" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:06.747247 11010 tablet_service.cc:1514] Processing DeleteTablet for tablet ab98a42082f34b839684b25d6386898b with delete_type TABLET_DATA_TOMBSTONED (TS 76827763179e465a92298cea85e53f2b not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:52404
I20240502 14:17:06.747598 11959 tablet_replica.cc:330] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: stopping tablet replica
I20240502 14:17:06.747778 11959 raft_consensus.cc:2229] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:06.748001 11959 raft_consensus.cc:2258] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:06.748972 11959 ts_tablet_manager.cc:1875] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:06.752559 10491 catalog_manager.cc:5484] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 reported cstate change: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.754979 11959 ts_tablet_manager.cc:1888] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:17:06.755154 11959 log.cc:1196] T ab98a42082f34b839684b25d6386898b P 76827763179e465a92298cea85e53f2b: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/ab98a42082f34b839684b25d6386898b
I20240502 14:17:06.755918 10482 catalog_manager.cc:4830] TS 76827763179e465a92298cea85e53f2b (127.27.220.132:35055): tablet ab98a42082f34b839684b25d6386898b (table rebalance_test_table_0 [id=8e56c7cd46804033b4ea083d1730d37b]) successfully deleted
I20240502 14:17:06.758435 11030 consensus_queue.cc:237] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } }
I20240502 14:17:06.757761 10637 raft_consensus.cc:2932] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 3be4f0789bca4f0483a4d3df81a8b0c3 (127.27.220.130) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.760073 11378 raft_consensus.cc:1261] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:06.760574 11885 consensus_queue.cc:1035] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:06.762971 11885 raft_consensus.cc:2932] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.763559 11378 raft_consensus.cc:2932] T e4876c504a97477cb27fb7f6c4ae12b6 P 3be4f0789bca4f0483a4d3df81a8b0c3 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } } }
I20240502 14:17:06.765920 10482 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e4876c504a97477cb27fb7f6c4ae12b6 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:06.767938 10490 catalog_manager.cc:5484] T e4876c504a97477cb27fb7f6c4ae12b6 P 76827763179e465a92298cea85e53f2b reported cstate change: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3be4f0789bca4f0483a4d3df81a8b0c3" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 33583 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:06.772089 10615 tablet_service.cc:1514] Processing DeleteTablet for tablet e4876c504a97477cb27fb7f6c4ae12b6 with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 5 (current committed config index is 6)) from {username='slave'} at 127.0.0.1:53314
I20240502 14:17:06.772473 11963 tablet_replica.cc:330] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: stopping tablet replica
I20240502 14:17:06.772665 11963 raft_consensus.cc:2229] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:06.772898 11963 raft_consensus.cc:2258] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:06.774073 11963 ts_tablet_manager.cc:1875] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:06.779523 11963 ts_tablet_manager.cc:1888] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:17:06.779636 11963 log.cc:1196] T e4876c504a97477cb27fb7f6c4ae12b6 P ced0bd81140c45c19192ed2611f0d58a: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/e4876c504a97477cb27fb7f6c4ae12b6
I20240502 14:17:06.780253 10482 catalog_manager.cc:4830] TS ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689): tablet e4876c504a97477cb27fb7f6c4ae12b6 (table rebalance_test_table_0 [id=8e56c7cd46804033b4ea083d1730d37b]) successfully deleted
I20240502 14:17:06.899003 11879 raft_consensus.cc:1050] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b: attempting to promote NON_VOTER 4224d6603b434d6cb7cf3b58a86a041f to VOTER
I20240502 14:17:06.899634 11879 consensus_queue.cc:237] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } }
I20240502 14:17:06.901842 10637 raft_consensus.cc:1261] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.901924 11509 raft_consensus.cc:1261] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 LEARNER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:06.902437 11879 consensus_queue.cc:1035] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.902772 11885 consensus_queue.cc:1035] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:06.905753 11879 raft_consensus.cc:2932] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.906419 11509 raft_consensus.cc:2932] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.906805 10637 raft_consensus.cc:2932] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.911692 10490 catalog_manager.cc:5484] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f reported cstate change: config changed from index 4 to 5, 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "ced0bd81140c45c19192ed2611f0d58a" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 37689 } attrs { replace: true } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.916935 11030 consensus_queue.cc:237] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } }
I20240502 14:17:06.918591 11509 raft_consensus.cc:1261] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Refusing update from remote peer 76827763179e465a92298cea85e53f2b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:06.919134 11884 consensus_queue.cc:1035] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [LEADER]: Connected to new peer: Peer: permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:06.921147 11885 raft_consensus.cc:2932] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.924026 10482 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e8980b56d2b04a2682f9e8c1b2c5e875 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:06.924181 11509 raft_consensus.cc:2932] T e8980b56d2b04a2682f9e8c1b2c5e875 P 4224d6603b434d6cb7cf3b58a86a041f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } } }
I20240502 14:17:06.925496 10491 catalog_manager.cc:5484] T e8980b56d2b04a2682f9e8c1b2c5e875 P 76827763179e465a92298cea85e53f2b reported cstate change: config changed from index 5 to 6, VOTER ced0bd81140c45c19192ed2611f0d58a (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "76827763179e465a92298cea85e53f2b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "76827763179e465a92298cea85e53f2b" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 35055 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4224d6603b434d6cb7cf3b58a86a041f" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 38943 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:06.929431 10615 tablet_service.cc:1514] Processing DeleteTablet for tablet e8980b56d2b04a2682f9e8c1b2c5e875 with delete_type TABLET_DATA_TOMBSTONED (TS ced0bd81140c45c19192ed2611f0d58a not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53314
I20240502 14:17:06.929805 11963 tablet_replica.cc:330] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: stopping tablet replica
I20240502 14:17:06.930004 11963 raft_consensus.cc:2229] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:06.930300 11963 raft_consensus.cc:2258] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:06.931313 11963 ts_tablet_manager.cc:1875] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:06.935787 11963 ts_tablet_manager.cc:1888] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 14:17:06.935899 11963 log.cc:1196] T e8980b56d2b04a2682f9e8c1b2c5e875 P ced0bd81140c45c19192ed2611f0d58a: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/e8980b56d2b04a2682f9e8c1b2c5e875
I20240502 14:17:06.936602 10482 catalog_manager.cc:4830] TS ced0bd81140c45c19192ed2611f0d58a (127.27.220.129:37689): tablet e8980b56d2b04a2682f9e8c1b2c5e875 (table rebalance_test_table_1 [id=672006a2715b45439494a055f37d9484]) successfully deleted
I20240502 14:17:07.104346 11489 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:07.107771 11010 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:07.110015 10615 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:17:07.116158 11358 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:07.276072 11358 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:07.278563 11489 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:07.279009 11010 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:07.295418 10615 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:17:07.428119 11358 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:07.438105 11010 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:07.440070 11489 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:07.448997 10615 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 14:17:07.631385 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 10555
I20240502 14:17:07.642989 10676 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:17:07.767489 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 10555
W20240502 14:17:07.790585 11311 connection.cc:535] server connection from 127.27.220.129:43949 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 14:17:07.791404 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 11295
I20240502 14:17:07.797487 11418 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:17:07.900867 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 11295
I20240502 14:17:07.929833 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 11427
I20240502 14:17:07.936722 11548 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:17:08.047143 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 11427
W20240502 14:17:08.067180 10967 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv got EOF from 127.27.220.131:38943 (error 108)
I20240502 14:17:08.067406 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 10948
W20240502 14:17:08.069413 10967 consensus_peers.cc:485] T f5830021cf7e4634995dc4782c2130e2 P 76827763179e465a92298cea85e53f2b -> Peer 4224d6603b434d6cb7cf3b58a86a041f (127.27.220.131:38943): Couldn't send request to peer 4224d6603b434d6cb7cf3b58a86a041f. Status: Network error: Client connection negotiation failed: client connection to 127.27.220.131:38943: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:08.074333 11071 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:17:08.198393 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 10948
I20240502 14:17:08.222144 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 10465
I20240502 14:17:08.223253 10526 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:17:08.352941 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 10465
2024-05-02T14:17:08Z chronyd exiting
[ OK ] LocationAwareRebalancingParamTest.Rebalance/idx7_rf2_t3_l_1_2_1 (14887 ms)
[ RUN ] LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer/idx6_rf7_t7_l_3_4_3
I20240502 14:17:08.403349 28530 test_util.cc:268] Using random seed: 1109657704
I20240502 14:17:08.405273 28530 ts_itest-base.cc:112] Starting cluster with:
I20240502 14:17:08.405347 28530 ts_itest-base.cc:113] --------------
I20240502 14:17:08.405395 28530 ts_itest-base.cc:114] 10 tablet servers
I20240502 14:17:08.405441 28530 ts_itest-base.cc:115] 7 replicas per TS
I20240502 14:17:08.405479 28530 ts_itest-base.cc:116] --------------
2024-05-02T14:17:08Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T14:17:08Z Disabled control of system clock
I20240502 14:17:08.450794 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:36663
--webserver_interface=127.27.220.190
--webserver_port=0
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.27.220.190:36663
--location_mapping_cmd=/tmp/dist-test-taskhB7nWc/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:4 --map /C:3
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false
--master_client_location_assignment_enabled=false
--allow_unsafe_replication_factor with env {}
W20240502 14:17:08.778869 12137 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 14:17:08.779211 12137 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:08.779286 12137 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:08.779388 12137 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:08.786571 12137 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:08.786679 12137 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 14:17:08.786775 12137 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:08.786832 12137 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 14:17:08.786911 12137 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 14:17:08.795717 12137 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-taskhB7nWc/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:4 --map /C:3
--ipki_ca_key_size=768
--master_addresses=127.27.220.190:36663
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.27.220.190:36663
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.27.220.190
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:08.796993 12137 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:08.798717 12137 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:08.809324 12142 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:08.809881 12145 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:08.810606 12143 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:08.811190 12137 server_base.cc:1034] running on GCE node
I20240502 14:17:08.811959 12137 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:08.813267 12137 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:08.814491 12137 hybrid_clock.cc:648] HybridClock initialized: now 1714659428814424 us; error 44 us; skew 500 ppm
I20240502 14:17:08.814772 12137 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:08.816797 12137 webserver.cc:460] Webserver started at http://127.27.220.190:46585/ using document root <none> and password file <none>
I20240502 14:17:08.817461 12137 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:08.817642 12137 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:08.817904 12137 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:08.820195 12137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "61c8566d69814e8195ceba499caccc6a"
format_stamp: "Formatted at 2024-05-02 14:17:08 on dist-test-slave-j72s"
I20240502 14:17:08.820905 12137 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "61c8566d69814e8195ceba499caccc6a"
format_stamp: "Formatted at 2024-05-02 14:17:08 on dist-test-slave-j72s"
I20240502 14:17:08.825672 12137 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.003s
I20240502 14:17:08.828802 12151 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:08.830053 12137 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:08.830200 12137 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
uuid: "61c8566d69814e8195ceba499caccc6a"
format_stamp: "Formatted at 2024-05-02 14:17:08 on dist-test-slave-j72s"
I20240502 14:17:08.830390 12137 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:08.843138 12137 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:08.843819 12137 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:08.844072 12137 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:08.861548 12137 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.190:36663
I20240502 14:17:08.861560 12202 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.190:36663 every 8 connection(s)
I20240502 14:17:08.862948 12137 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 14:17:08.865079 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 12137
I20240502 14:17:08.865331 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 14:17:08.866830 12203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:08.879572 12203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a: Bootstrap starting.
I20240502 14:17:08.882697 12203 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:08.883739 12203 log.cc:824] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:08.886256 12203 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a: No bootstrap required, opened a new log
I20240502 14:17:08.892158 12203 raft_consensus.cc:348] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "61c8566d69814e8195ceba499caccc6a" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 36663 } }
I20240502 14:17:08.892494 12203 raft_consensus.cc:374] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:08.892594 12203 raft_consensus.cc:729] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 61c8566d69814e8195ceba499caccc6a, State: Initialized, Role: FOLLOWER
I20240502 14:17:08.893134 12203 consensus_queue.cc:260] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "61c8566d69814e8195ceba499caccc6a" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 36663 } }
I20240502 14:17:08.893337 12203 raft_consensus.cc:388] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 14:17:08.893471 12203 raft_consensus.cc:482] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 14:17:08.893672 12203 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:08.895664 12203 raft_consensus.cc:504] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "61c8566d69814e8195ceba499caccc6a" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 36663 } }
I20240502 14:17:08.896111 12203 leader_election.cc:302] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 61c8566d69814e8195ceba499caccc6a; no voters:
I20240502 14:17:08.896664 12203 leader_election.cc:288] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 14:17:08.896786 12208 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:08.897398 12208 raft_consensus.cc:686] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [term 1 LEADER]: Becoming Leader. State: Replica: 61c8566d69814e8195ceba499caccc6a, State: Running, Role: LEADER
I20240502 14:17:08.898000 12208 consensus_queue.cc:237] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "61c8566d69814e8195ceba499caccc6a" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 36663 } }
I20240502 14:17:08.898612 12203 sys_catalog.cc:564] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [sys.catalog]: configured and running, proceeding with master startup.
I20240502 14:17:08.900750 12209 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "61c8566d69814e8195ceba499caccc6a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "61c8566d69814e8195ceba499caccc6a" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 36663 } } }
I20240502 14:17:08.901154 12209 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [sys.catalog]: This master's current role is: LEADER
I20240502 14:17:08.900820 12210 sys_catalog.cc:455] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [sys.catalog]: SysCatalogTable state changed. Reason: New leader 61c8566d69814e8195ceba499caccc6a. Latest consensus state: current_term: 1 leader_uuid: "61c8566d69814e8195ceba499caccc6a" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "61c8566d69814e8195ceba499caccc6a" member_type: VOTER last_known_addr { host: "127.27.220.190" port: 36663 } } }
I20240502 14:17:08.902599 12210 sys_catalog.cc:458] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a [sys.catalog]: This master's current role is: LEADER
I20240502 14:17:08.903337 12214 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 14:17:08.908527 12214 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 14:17:08.919934 12214 catalog_manager.cc:1347] Generated new cluster ID: 6893ffeb72324e38a41b8512a565b03f
I20240502 14:17:08.920065 12214 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 14:17:08.930130 12214 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 14:17:08.931288 12214 catalog_manager.cc:1504] Loading token signing keys...
I20240502 14:17:08.942556 12214 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 61c8566d69814e8195ceba499caccc6a: Generated new TSK 0
I20240502 14:17:08.943382 12214 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 14:17:08.958626 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.129:0
--local_ip_for_outbound_sockets=127.27.220.129
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:17:09.306388 12227 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:09.306694 12227 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:09.306860 12227 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:09.314728 12227 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:09.314867 12227 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:09.315040 12227 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.129
I20240502 14:17:09.324244 12227 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.27.220.129
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.129
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:09.325649 12227 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:09.327455 12227 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:09.340442 12232 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:09.340972 12233 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:09.341640 12235 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:09.342255 12227 server_base.cc:1034] running on GCE node
I20240502 14:17:09.342803 12227 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:09.343724 12227 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:09.344925 12227 hybrid_clock.cc:648] HybridClock initialized: now 1714659429344876 us; error 44 us; skew 500 ppm
I20240502 14:17:09.345224 12227 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:09.347518 12227 webserver.cc:460] Webserver started at http://127.27.220.129:33875/ using document root <none> and password file <none>
I20240502 14:17:09.348218 12227 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:09.348331 12227 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:09.348634 12227 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:09.351061 12227 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "70a425a77fe54cbcb9755f9c540fd367"
format_stamp: "Formatted at 2024-05-02 14:17:09 on dist-test-slave-j72s"
I20240502 14:17:09.351766 12227 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "70a425a77fe54cbcb9755f9c540fd367"
format_stamp: "Formatted at 2024-05-02 14:17:09 on dist-test-slave-j72s"
I20240502 14:17:09.356580 12227 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.001s sys 0.005s
I20240502 14:17:09.359532 12241 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:09.360702 12227 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:09.360847 12227 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "70a425a77fe54cbcb9755f9c540fd367"
format_stamp: "Formatted at 2024-05-02 14:17:09 on dist-test-slave-j72s"
I20240502 14:17:09.361052 12227 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:09.380259 12227 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:09.381126 12227 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:09.381385 12227 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:09.382328 12227 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:09.383738 12227 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:09.383836 12227 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:09.383951 12227 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:09.384028 12227 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:09.414707 12227 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.129:46613
I20240502 14:17:09.414754 12353 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.129:46613 every 8 connection(s)
I20240502 14:17:09.416016 12227 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 14:17:09.422384 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 12227
I20240502 14:17:09.422626 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 14:17:09.426504 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.130:0
--local_ip_for_outbound_sockets=127.27.220.130
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:09.428980 12354 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:09.429255 12354 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:09.430090 12354 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:09.480811 12168 ts_manager.cc:194] Registered new tserver with Master: 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:09.483342 12168 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.129:50705
W20240502 14:17:09.774859 12358 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:09.775138 12358 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:09.775229 12358 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:09.782532 12358 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:09.782649 12358 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:09.782770 12358 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.130
I20240502 14:17:09.791321 12358 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.27.220.130
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.130
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:09.792627 12358 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:09.794314 12358 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:09.805886 12365 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:09.806370 12364 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:09.806959 12367 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:09.807557 12358 server_base.cc:1034] running on GCE node
I20240502 14:17:09.808066 12358 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:09.808868 12358 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:09.810057 12358 hybrid_clock.cc:648] HybridClock initialized: now 1714659429810003 us; error 49 us; skew 500 ppm
I20240502 14:17:09.810331 12358 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:09.824144 12358 webserver.cc:460] Webserver started at http://127.27.220.130:37289/ using document root <none> and password file <none>
I20240502 14:17:09.825053 12358 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:09.825212 12358 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:09.825598 12358 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:09.830235 12358 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "eddeff583cf34c8dab2942fb36abbf9e"
format_stamp: "Formatted at 2024-05-02 14:17:09 on dist-test-slave-j72s"
I20240502 14:17:09.831180 12358 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "eddeff583cf34c8dab2942fb36abbf9e"
format_stamp: "Formatted at 2024-05-02 14:17:09 on dist-test-slave-j72s"
I20240502 14:17:09.837932 12358 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.007s sys 0.000s
I20240502 14:17:09.841881 12373 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:09.843323 12358 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 14:17:09.843510 12358 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "eddeff583cf34c8dab2942fb36abbf9e"
format_stamp: "Formatted at 2024-05-02 14:17:09 on dist-test-slave-j72s"
I20240502 14:17:09.843699 12358 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:09.881263 12358 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:09.882297 12358 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:09.882547 12358 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:09.883479 12358 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:09.884826 12358 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:09.884907 12358 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:09.884980 12358 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:09.885028 12358 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:09.916713 12358 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.130:34491
I20240502 14:17:09.916786 12485 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.130:34491 every 8 connection(s)
I20240502 14:17:09.918080 12358 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 14:17:09.922742 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 12358
I20240502 14:17:09.922982 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 14:17:09.926468 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.131:0
--local_ip_for_outbound_sockets=127.27.220.131
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:09.931816 12486 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:09.932220 12486 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:09.933344 12486 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:09.982053 12168 ts_manager.cc:194] Registered new tserver with Master: eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:09.982975 12168 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.130:56757
W20240502 14:17:10.255852 12489 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:10.256166 12489 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:10.256299 12489 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:10.263430 12489 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:10.263569 12489 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:10.263742 12489 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.131
I20240502 14:17:10.272637 12489 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.27.220.131
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.131
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:10.273980 12489 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:10.275710 12489 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:10.287662 12495 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:10.288383 12496 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:10.288812 12498 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:10.289388 12489 server_base.cc:1034] running on GCE node
I20240502 14:17:10.290185 12489 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:10.291132 12489 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:10.292362 12489 hybrid_clock.cc:648] HybridClock initialized: now 1714659430292300 us; error 49 us; skew 500 ppm
I20240502 14:17:10.292694 12489 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:10.294953 12489 webserver.cc:460] Webserver started at http://127.27.220.131:36575/ using document root <none> and password file <none>
I20240502 14:17:10.295691 12489 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:10.295800 12489 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:10.296131 12489 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:10.298856 12489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "53479f0fb09c454e9b3cbe7398bf8874"
format_stamp: "Formatted at 2024-05-02 14:17:10 on dist-test-slave-j72s"
I20240502 14:17:10.299603 12489 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "53479f0fb09c454e9b3cbe7398bf8874"
format_stamp: "Formatted at 2024-05-02 14:17:10 on dist-test-slave-j72s"
I20240502 14:17:10.304852 12489 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 14:17:10.308136 12504 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:10.309710 12489 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
I20240502 14:17:10.309875 12489 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "53479f0fb09c454e9b3cbe7398bf8874"
format_stamp: "Formatted at 2024-05-02 14:17:10 on dist-test-slave-j72s"
I20240502 14:17:10.310088 12489 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:10.328979 12489 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:10.329929 12489 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:10.330221 12489 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:10.331167 12489 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:10.332674 12489 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:10.332777 12489 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:10.332901 12489 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:10.332983 12489 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:10.366228 12489 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.131:43727
I20240502 14:17:10.366287 12616 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.131:43727 every 8 connection(s)
I20240502 14:17:10.367558 12489 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 14:17:10.370296 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 12489
I20240502 14:17:10.370492 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 14:17:10.373744 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.132:0
--local_ip_for_outbound_sockets=127.27.220.132
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:10.380970 12617 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:10.381323 12617 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:10.382447 12617 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:10.430686 12168 ts_manager.cc:194] Registered new tserver with Master: 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:10.431686 12168 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.131:57715
I20240502 14:17:10.486919 12354 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
W20240502 14:17:10.714648 12620 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:10.714943 12620 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:10.715093 12620 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:10.722471 12620 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:10.722595 12620 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:10.722755 12620 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.132
I20240502 14:17:10.731492 12620 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.132:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.27.220.132
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.132
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:10.732880 12620 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:10.734735 12620 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:10.746621 12627 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:10.747167 12626 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:10.747748 12629 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:10.748420 12620 server_base.cc:1034] running on GCE node
I20240502 14:17:10.749037 12620 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:10.750026 12620 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:10.751226 12620 hybrid_clock.cc:648] HybridClock initialized: now 1714659430751182 us; error 39 us; skew 500 ppm
I20240502 14:17:10.751540 12620 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:10.753849 12620 webserver.cc:460] Webserver started at http://127.27.220.132:46853/ using document root <none> and password file <none>
I20240502 14:17:10.754582 12620 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:10.754700 12620 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:10.755012 12620 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:10.757580 12620 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "af704d4ab2f2496a833c692bd3d25fde"
format_stamp: "Formatted at 2024-05-02 14:17:10 on dist-test-slave-j72s"
I20240502 14:17:10.758304 12620 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "af704d4ab2f2496a833c692bd3d25fde"
format_stamp: "Formatted at 2024-05-02 14:17:10 on dist-test-slave-j72s"
I20240502 14:17:10.763103 12620 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
I20240502 14:17:10.766517 12635 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:10.767729 12620 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:10.767882 12620 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "af704d4ab2f2496a833c692bd3d25fde"
format_stamp: "Formatted at 2024-05-02 14:17:10 on dist-test-slave-j72s"
I20240502 14:17:10.768059 12620 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:10.785684 12620 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:10.786556 12620 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:10.786840 12620 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:10.787882 12620 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:10.789394 12620 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:10.789492 12620 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:10.789678 12620 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:10.789762 12620 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:10.823206 12620 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.132:45069
I20240502 14:17:10.823253 12747 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.132:45069 every 8 connection(s)
I20240502 14:17:10.824494 12620 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 14:17:10.825096 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 12620
I20240502 14:17:10.825311 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 14:17:10.828742 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.133:0
--local_ip_for_outbound_sockets=127.27.220.133
--webserver_interface=127.27.220.133
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:10.838191 12748 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:10.838480 12748 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:10.839295 12748 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:10.886277 12168 ts_manager.cc:194] Registered new tserver with Master: af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069)
I20240502 14:17:10.887442 12168 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.132:52157
I20240502 14:17:10.985709 12486 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
W20240502 14:17:11.177110 12751 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:11.177397 12751 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:11.177487 12751 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:11.184827 12751 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:11.184940 12751 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:11.185055 12751 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.133
I20240502 14:17:11.193626 12751 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.133:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.27.220.133
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.133
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:11.194834 12751 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:11.196491 12751 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:11.208058 12757 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:11.208513 12758 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:11.209095 12760 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:11.209834 12751 server_base.cc:1034] running on GCE node
I20240502 14:17:11.210389 12751 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:11.211280 12751 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:11.212502 12751 hybrid_clock.cc:648] HybridClock initialized: now 1714659431212454 us; error 40 us; skew 500 ppm
I20240502 14:17:11.212800 12751 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:11.215121 12751 webserver.cc:460] Webserver started at http://127.27.220.133:37229/ using document root <none> and password file <none>
I20240502 14:17:11.215828 12751 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:11.215946 12751 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:11.216244 12751 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:11.218815 12751 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
format_stamp: "Formatted at 2024-05-02 14:17:11 on dist-test-slave-j72s"
I20240502 14:17:11.219503 12751 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
format_stamp: "Formatted at 2024-05-02 14:17:11 on dist-test-slave-j72s"
I20240502 14:17:11.224382 12751 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 14:17:11.227694 12766 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:11.229295 12751 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:11.229460 12751 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
format_stamp: "Formatted at 2024-05-02 14:17:11 on dist-test-slave-j72s"
I20240502 14:17:11.229694 12751 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:11.251637 12751 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:11.252532 12751 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:11.252804 12751 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:11.253796 12751 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:11.255270 12751 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:11.255367 12751 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:11.255475 12751 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:11.255549 12751 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:11.286394 12751 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.133:45069
I20240502 14:17:11.286444 12878 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.133:45069 every 8 connection(s)
I20240502 14:17:11.287744 12751 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 14:17:11.292812 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 12751
I20240502 14:17:11.292999 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 14:17:11.296036 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.134:0
--local_ip_for_outbound_sockets=127.27.220.134
--webserver_interface=127.27.220.134
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:11.301017 12879 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:11.301303 12879 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:11.302152 12879 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:11.347852 12168 ts_manager.cc:194] Registered new tserver with Master: c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069)
I20240502 14:17:11.348759 12168 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.133:57965
I20240502 14:17:11.434707 12617 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
W20240502 14:17:11.629686 12882 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:11.629932 12882 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:11.630049 12882 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:11.637137 12882 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:11.637256 12882 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:11.637413 12882 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.134
I20240502 14:17:11.645268 12882 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.134:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.27.220.134
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.134
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:11.646350 12882 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:11.647984 12882 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:11.659857 12889 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:11.660260 12891 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:11.660609 12888 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:11.661276 12882 server_base.cc:1034] running on GCE node
I20240502 14:17:11.661798 12882 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:11.662631 12882 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:11.663825 12882 hybrid_clock.cc:648] HybridClock initialized: now 1714659431663785 us; error 34 us; skew 500 ppm
I20240502 14:17:11.664103 12882 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:11.666188 12882 webserver.cc:460] Webserver started at http://127.27.220.134:41357/ using document root <none> and password file <none>
I20240502 14:17:11.666841 12882 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:11.666947 12882 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:11.667222 12882 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:11.669585 12882 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "d693e089cfa54fbb8df91939410b19db"
format_stamp: "Formatted at 2024-05-02 14:17:11 on dist-test-slave-j72s"
I20240502 14:17:11.670313 12882 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "d693e089cfa54fbb8df91939410b19db"
format_stamp: "Formatted at 2024-05-02 14:17:11 on dist-test-slave-j72s"
I20240502 14:17:11.675052 12882 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
I20240502 14:17:11.678035 12897 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:11.679138 12882 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 14:17:11.679275 12882 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "d693e089cfa54fbb8df91939410b19db"
format_stamp: "Formatted at 2024-05-02 14:17:11 on dist-test-slave-j72s"
I20240502 14:17:11.679454 12882 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:11.692127 12882 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:11.692792 12882 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:11.693019 12882 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:11.693902 12882 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:11.695215 12882 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:11.695298 12882 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:11.695385 12882 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:11.695463 12882 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:11.725026 12882 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.134:36329
I20240502 14:17:11.725075 13009 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.134:36329 every 8 connection(s)
I20240502 14:17:11.726250 12882 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 14:17:11.736377 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 12882
I20240502 14:17:11.736557 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 14:17:11.738582 13010 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:11.738835 13010 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:11.739679 13010 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:11.740140 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.135:0
--local_ip_for_outbound_sockets=127.27.220.135
--webserver_interface=127.27.220.135
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:11.786375 12167 ts_manager.cc:194] Registered new tserver with Master: d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:11.787355 12167 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.134:52293
W20240502 14:17:11.825484 12744 messenger.cc:188] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.27.220.132:45069
I20240502 14:17:11.890682 12748 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
W20240502 14:17:12.081475 13014 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:12.081815 13014 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:12.081962 13014 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:12.089668 13014 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:12.089803 13014 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:12.089967 13014 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.135
I20240502 14:17:12.098649 13014 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.135:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.27.220.135
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.135
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:12.100008 13014 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:12.101698 13014 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:12.113463 13019 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:12.113588 13020 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:12.114331 13014 server_base.cc:1034] running on GCE node
W20240502 14:17:12.114845 13022 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:12.115500 13014 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:12.116443 13014 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:12.117611 13014 hybrid_clock.cc:648] HybridClock initialized: now 1714659432117565 us; error 35 us; skew 500 ppm
I20240502 14:17:12.117923 13014 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:12.119987 13014 webserver.cc:460] Webserver started at http://127.27.220.135:37945/ using document root <none> and password file <none>
I20240502 14:17:12.120748 13014 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:12.120887 13014 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:12.121177 13014 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:12.123703 13014 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
format_stamp: "Formatted at 2024-05-02 14:17:12 on dist-test-slave-j72s"
I20240502 14:17:12.124401 13014 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
format_stamp: "Formatted at 2024-05-02 14:17:12 on dist-test-slave-j72s"
I20240502 14:17:12.129181 13014 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20240502 14:17:12.132398 13028 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:12.133733 13014 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:12.133929 13014 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
format_stamp: "Formatted at 2024-05-02 14:17:12 on dist-test-slave-j72s"
I20240502 14:17:12.134138 13014 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:12.150658 13014 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:12.151455 13014 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:12.151707 13014 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:12.152654 13014 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:12.154088 13014 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:12.154187 13014 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:12.154300 13014 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:12.154371 13014 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:12.184692 13014 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.135:44203
I20240502 14:17:12.184746 13140 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.135:44203 every 8 connection(s)
I20240502 14:17:12.185961 13014 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 14:17:12.192178 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 13014
I20240502 14:17:12.192317 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 14:17:12.195461 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.136:0
--local_ip_for_outbound_sockets=127.27.220.136
--webserver_interface=127.27.220.136
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:12.198447 13141 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:12.198781 13141 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:12.199759 13141 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:12.288082 12167 ts_manager.cc:194] Registered new tserver with Master: 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:12.289428 12167 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.135:53895
W20240502 14:17:12.289404 12875 messenger.cc:188] Invalid argument: socket address is ambiguous: no data on pending connections for acceptor pool at 127.27.220.133:45069
I20240502 14:17:12.351502 12879 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
W20240502 14:17:12.559803 13144 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:12.560108 13144 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:12.560259 13144 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:12.567692 13144 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:12.567826 13144 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:12.567986 13144 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.136
I20240502 14:17:12.577646 13144 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.136:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.27.220.136
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.136
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:12.579025 13144 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:12.580806 13144 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:12.592700 13150 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:12.593333 13153 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:12.593830 13151 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:12.594635 13144 server_base.cc:1034] running on GCE node
I20240502 14:17:12.595213 13144 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:12.596123 13144 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:12.597347 13144 hybrid_clock.cc:648] HybridClock initialized: now 1714659432597296 us; error 40 us; skew 500 ppm
I20240502 14:17:12.597695 13144 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:12.600028 13144 webserver.cc:460] Webserver started at http://127.27.220.136:33735/ using document root <none> and password file <none>
I20240502 14:17:12.600728 13144 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:12.600833 13144 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:12.601158 13144 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:12.603824 13144 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "4c04a4a8d8864267be7f67c2abe98f84"
format_stamp: "Formatted at 2024-05-02 14:17:12 on dist-test-slave-j72s"
I20240502 14:17:12.604548 13144 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "4c04a4a8d8864267be7f67c2abe98f84"
format_stamp: "Formatted at 2024-05-02 14:17:12 on dist-test-slave-j72s"
I20240502 14:17:12.609596 13144 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.002s
I20240502 14:17:12.612783 13159 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:12.614136 13144 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.003s
I20240502 14:17:12.614310 13144 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "4c04a4a8d8864267be7f67c2abe98f84"
format_stamp: "Formatted at 2024-05-02 14:17:12 on dist-test-slave-j72s"
I20240502 14:17:12.614516 13144 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:12.634735 13144 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:12.635701 13144 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:12.636011 13144 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:12.636993 13144 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:12.638712 13144 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:12.638819 13144 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:12.638932 13144 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:12.639014 13144 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:12.671398 13144 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.136:41465
I20240502 14:17:12.671447 13271 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.136:41465 every 8 connection(s)
I20240502 14:17:12.672811 13144 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 14:17:12.677650 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 13144
I20240502 14:17:12.677850 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 14:17:12.681232 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.137:0
--local_ip_for_outbound_sockets=127.27.220.137
--webserver_interface=127.27.220.137
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:12.686664 13272 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:12.686976 13272 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:12.687924 13272 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:12.750000 12167 ts_manager.cc:194] Registered new tserver with Master: 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465)
I20240502 14:17:12.751024 12167 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.136:60901
I20240502 14:17:12.790611 13010 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
W20240502 14:17:13.034052 13275 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:13.034344 13275 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:13.034464 13275 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:13.042111 13275 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:13.042261 13275 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:13.042392 13275 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.137
I20240502 14:17:13.051368 13275 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.137:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.27.220.137
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.137
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:13.052683 13275 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:13.054389 13275 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:13.066715 13284 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:13.066902 13282 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:13.067524 13281 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:13.068164 13275 server_base.cc:1034] running on GCE node
I20240502 14:17:13.068677 13275 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:13.069545 13275 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:13.070742 13275 hybrid_clock.cc:648] HybridClock initialized: now 1714659433070694 us; error 44 us; skew 500 ppm
I20240502 14:17:13.071039 13275 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:13.073401 13275 webserver.cc:460] Webserver started at http://127.27.220.137:38207/ using document root <none> and password file <none>
I20240502 14:17:13.074124 13275 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:13.074219 13275 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:13.074501 13275 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:13.077075 13275 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/instance:
uuid: "95e58ca33b4048e2b000dfbf2b5a0186"
format_stamp: "Formatted at 2024-05-02 14:17:13 on dist-test-slave-j72s"
I20240502 14:17:13.077836 13275 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal/instance:
uuid: "95e58ca33b4048e2b000dfbf2b5a0186"
format_stamp: "Formatted at 2024-05-02 14:17:13 on dist-test-slave-j72s"
I20240502 14:17:13.082648 13275 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.002s sys 0.002s
I20240502 14:17:13.085768 13290 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:13.086999 13275 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 14:17:13.087131 13275 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "95e58ca33b4048e2b000dfbf2b5a0186"
format_stamp: "Formatted at 2024-05-02 14:17:13 on dist-test-slave-j72s"
I20240502 14:17:13.087316 13275 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:13.109325 13275 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:13.110071 13275 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:13.110360 13275 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:13.111225 13275 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:13.112663 13275 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:13.112749 13275 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:13.112833 13275 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:13.112897 13275 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:13.143666 13275 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.137:40043
I20240502 14:17:13.143755 13402 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.137:40043 every 8 connection(s)
I20240502 14:17:13.144999 13275 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240502 14:17:13.147802 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 13275
I20240502 14:17:13.147955 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal/instance
I20240502 14:17:13.151160 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.138:0
--local_ip_for_outbound_sockets=127.27.220.138
--webserver_interface=127.27.220.138
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:13.158576 13403 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:13.158914 13403 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:13.159937 13403 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:13.220858 12167 ts_manager.cc:194] Registered new tserver with Master: 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043)
I20240502 14:17:13.221863 12167 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.137:55829
I20240502 14:17:13.292778 13141 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
W20240502 14:17:13.518837 13406 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:13.519129 13406 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:13.519281 13406 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:13.526482 13406 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:13.526618 13406 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:13.526757 13406 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.138
I20240502 14:17:13.535585 13406 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.138:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--webserver_interface=127.27.220.138
--webserver_port=0
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.138
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:13.536816 13406 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:13.538507 13406 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:13.549719 13412 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:13.550163 13415 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:13.550828 13413 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:13.551481 13406 server_base.cc:1034] running on GCE node
I20240502 14:17:13.551982 13406 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:13.552871 13406 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:13.554075 13406 hybrid_clock.cc:648] HybridClock initialized: now 1714659433554017 us; error 46 us; skew 500 ppm
I20240502 14:17:13.554369 13406 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:13.556354 13406 webserver.cc:460] Webserver started at http://127.27.220.138:40151/ using document root <none> and password file <none>
I20240502 14:17:13.557008 13406 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:13.557102 13406 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:13.557402 13406 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 14:17:13.559702 13406 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/instance:
uuid: "1ba6a238e6ad4e4a84eff746ff336144"
format_stamp: "Formatted at 2024-05-02 14:17:13 on dist-test-slave-j72s"
I20240502 14:17:13.560386 13406 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal/instance:
uuid: "1ba6a238e6ad4e4a84eff746ff336144"
format_stamp: "Formatted at 2024-05-02 14:17:13 on dist-test-slave-j72s"
I20240502 14:17:13.565181 13406 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20240502 14:17:13.568310 13421 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:13.569415 13406 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 14:17:13.569597 13406 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
uuid: "1ba6a238e6ad4e4a84eff746ff336144"
format_stamp: "Formatted at 2024-05-02 14:17:13 on dist-test-slave-j72s"
I20240502 14:17:13.569777 13406 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:13.589377 13406 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:13.590096 13406 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:13.590332 13406 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:13.591301 13406 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:13.592965 13406 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:13.593065 13406 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:13.593175 13406 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:13.593250 13406 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:13.623293 13406 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.138:33073
I20240502 14:17:13.623340 13533 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.138:33073 every 8 connection(s)
I20240502 14:17:13.624444 13406 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/info.pb
I20240502 14:17:13.627795 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 13406
I20240502 14:17:13.627970 28530 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal/instance
I20240502 14:17:13.636690 13534 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:13.636924 13534 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:13.637622 13534 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:13.681097 12167 ts_manager.cc:194] Registered new tserver with Master: 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073)
I20240502 14:17:13.682047 12167 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.138:55259
I20240502 14:17:13.684013 28530 external_mini_cluster.cc:930] 10 TS(s) registered with all masters
I20240502 14:17:13.700531 28530 external_mini_cluster.cc:930] 10 TS(s) registered with all masters
I20240502 14:17:13.700654 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 13144
I20240502 14:17:13.708215 13267 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:17:13.786446 13272 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
I20240502 14:17:13.788110 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 13144
I20240502 14:17:13.804880 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 13406
I20240502 14:17:13.813143 13529 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:17:13.890416 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 13406
I20240502 14:17:13.907130 28530 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu pid 13275
I20240502 14:17:13.915256 13398 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 14:17:13.995411 28530 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu with pid 13275
I20240502 14:17:17.766561 12167 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60530:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:17.773602 12167 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 14:17:17.820035 12552 tablet_service.cc:1467] Processing CreateTablet for tablet cd382683ec2b4600b6edaced79698b7e (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.820871 12550 tablet_service.cc:1467] Processing CreateTablet for tablet 00aebb2064324c3eac8145d1c455622c (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.822080 12550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.824254 12421 tablet_service.cc:1467] Processing CreateTablet for tablet cd382683ec2b4600b6edaced79698b7e (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.820101 12551 tablet_service.cc:1467] Processing CreateTablet for tablet b33c5e313e784b97987eb07b04f615f8 (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.826036 12421 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.826112 12551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.829974 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.834525 12420 tablet_service.cc:1467] Processing CreateTablet for tablet b33c5e313e784b97987eb07b04f615f8 (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.833881 12814 tablet_service.cc:1467] Processing CreateTablet for tablet cd382683ec2b4600b6edaced79698b7e (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.835065 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.835845 12814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.838045 12419 tablet_service.cc:1467] Processing CreateTablet for tablet 00aebb2064324c3eac8145d1c455622c (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.838531 12419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.842851 12812 tablet_service.cc:1467] Processing CreateTablet for tablet 00aebb2064324c3eac8145d1c455622c (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.843400 12812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.843916 12813 tablet_service.cc:1467] Processing CreateTablet for tablet b33c5e313e784b97987eb07b04f615f8 (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.844409 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.854175 12682 tablet_service.cc:1467] Processing CreateTablet for tablet b33c5e313e784b97987eb07b04f615f8 (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.854279 12683 tablet_service.cc:1467] Processing CreateTablet for tablet cd382683ec2b4600b6edaced79698b7e (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.855867 12682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.858904 12945 tablet_service.cc:1467] Processing CreateTablet for tablet cd382683ec2b4600b6edaced79698b7e (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.860412 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.860878 12681 tablet_service.cc:1467] Processing CreateTablet for tablet 00aebb2064324c3eac8145d1c455622c (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.861411 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.861934 12683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.876217 12289 tablet_service.cc:1467] Processing CreateTablet for tablet cd382683ec2b4600b6edaced79698b7e (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.878190 12287 tablet_service.cc:1467] Processing CreateTablet for tablet 00aebb2064324c3eac8145d1c455622c (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.879216 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.879144 12944 tablet_service.cc:1467] Processing CreateTablet for tablet b33c5e313e784b97987eb07b04f615f8 (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.879705 12944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.880038 13568 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:17.880986 12943 tablet_service.cc:1467] Processing CreateTablet for tablet 00aebb2064324c3eac8145d1c455622c (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.881395 12943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.881961 12288 tablet_service.cc:1467] Processing CreateTablet for tablet b33c5e313e784b97987eb07b04f615f8 (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.882530 12288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.885294 13568 tablet_bootstrap.cc:654] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:17.886641 13568 log.cc:824] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:17.893807 12289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.894008 13570 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:17.898362 13570 tablet_bootstrap.cc:654] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:17.910434 13570 log.cc:824] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:17.939332 13566 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:17.944456 13566 tablet_bootstrap.cc:654] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:17.950306 13573 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:17.959936 13572 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:17.960599 13076 tablet_service.cc:1467] Processing CreateTablet for tablet cd382683ec2b4600b6edaced79698b7e (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.962216 13076 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.964385 13572 tablet_bootstrap.cc:654] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:17.965833 13074 tablet_service.cc:1467] Processing CreateTablet for tablet 00aebb2064324c3eac8145d1c455622c (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.966024 13572 log.cc:824] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:17.966327 13074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.967036 13573 tablet_bootstrap.cc:654] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:17.967228 13075 tablet_service.cc:1467] Processing CreateTablet for tablet b33c5e313e784b97987eb07b04f615f8 (DEFAULT_TABLE table=rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:17.967803 13075 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:17.968567 13573 log.cc:824] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:17.976764 13572 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:17.977144 13572 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.017s user 0.007s sys 0.007s
I20240502 14:17:17.978935 13566 log.cc:824] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:17.980556 13567 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:17.983263 13570 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:17.983566 13570 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.090s user 0.016s sys 0.012s
I20240502 14:17:17.988754 13567 tablet_bootstrap.cc:654] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:17.989398 13572 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:17.990093 13572 raft_consensus.cc:374] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:17.990080 13567 log.cc:824] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:17.992825 13566 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:17.993232 13566 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.054s user 0.010s sys 0.000s
I20240502 14:17:17.993570 13570 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:17.995182 13570 raft_consensus.cc:374] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:17.995386 13570 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:17.996534 13570 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:17.997632 13568 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:17.997997 13568 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.118s user 0.047s sys 0.021s
I20240502 14:17:17.998922 13572 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:17.999928 13567 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:18.000257 13567 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.020s user 0.011s sys 0.004s
I20240502 14:17:18.001017 13570 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.017s user 0.008s sys 0.004s
I20240502 14:17:18.001427 13570 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:18.003834 13572 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.003212 13570 tablet_bootstrap.cc:654] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.004521 13578 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:18.006125 13573 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:18.006474 13573 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.057s user 0.009s sys 0.000s
I20240502 14:17:18.008159 13578 tablet_bootstrap.cc:654] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.009420 13578 log.cc:824] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:18.012466 13566 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.013322 13568 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.014372 13568 raft_consensus.cc:374] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.014485 13568 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.015404 13568 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.015261 13567 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.015949 13567 raft_consensus.cc:374] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.016075 13567 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.017093 13567 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.016884 13566 raft_consensus.cc:374] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.017841 13566 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.019691 13570 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:18.019860 13570 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.019s user 0.003s sys 0.012s
I20240502 14:17:18.020789 13572 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.043s user 0.010s sys 0.003s
I20240502 14:17:18.021380 13572 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:18.022851 13570 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.023154 13570 raft_consensus.cc:374] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.023267 13570 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.023551 13570 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.024240 13570 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240502 14:17:18.024950 13567 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.025s user 0.013s sys 0.004s
I20240502 14:17:18.025383 13567 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:18.024529 13570 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:18.027873 13570 tablet_bootstrap.cc:654] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.028128 13567 tablet_bootstrap.cc:654] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.040925 13573 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.034725 13566 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.041668 13573 raft_consensus.cc:374] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.041860 13573 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.042681 13572 tablet_bootstrap.cc:654] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.042970 13573 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
W20240502 14:17:18.044687 12880 tablet.cc:2369] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:17:18.045020 13567 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:18.045225 13567 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.020s user 0.008s sys 0.008s
I20240502 14:17:18.046499 13567 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.046792 13567 raft_consensus.cc:374] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.046916 13567 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.047212 13567 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.047500 13570 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:18.047688 13570 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.023s user 0.005s sys 0.010s
I20240502 14:17:18.047817 13567 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 14:17:18.048106 13567 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:18.048382 13578 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:18.048527 13570 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.048856 13570 raft_consensus.cc:374] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.048897 13578 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.044s user 0.012s sys 0.008s
I20240502 14:17:18.048949 13570 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.049224 13570 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.049881 13567 tablet_bootstrap.cc:654] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.050072 13570 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 14:17:18.052152 13568 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.054s user 0.007s sys 0.017s
I20240502 14:17:18.052672 13568 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:18.054531 13568 tablet_bootstrap.cc:654] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.056303 13566 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.063s user 0.007s sys 0.004s
I20240502 14:17:18.056583 13567 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:18.056778 13567 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.000s
I20240502 14:17:18.056989 13566 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:18.057576 13567 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.057941 13567 raft_consensus.cc:374] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.058018 13567 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.058302 13567 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.058934 13567 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:18.063484 13568 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:18.063674 13568 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.011s user 0.010s sys 0.000s
I20240502 14:17:18.064230 13566 tablet_bootstrap.cc:654] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.064775 13568 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.065183 13568 raft_consensus.cc:374] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.065296 13568 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.065727 13568 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.066448 13568 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:18.066798 13568 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:18.067551 13572 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:18.067781 13572 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.047s user 0.013s sys 0.003s
I20240502 14:17:18.068908 13568 tablet_bootstrap.cc:654] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.068861 13572 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.069250 13572 raft_consensus.cc:374] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.069387 13572 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.069801 13572 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.069893 13578 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.070425 13578 raft_consensus.cc:374] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.070554 13578 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.070586 13572 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:18.071010 13572 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:18.071389 13578 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.073145 13572 tablet_bootstrap.cc:654] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
W20240502 14:17:18.081806 12749 tablet.cc:2369] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:17:18.081907 13573 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.075s user 0.009s sys 0.009s
I20240502 14:17:18.085031 13573 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:18.087356 13573 tablet_bootstrap.cc:654] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.089397 13566 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:18.089658 13566 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.033s user 0.008s sys 0.003s
I20240502 14:17:18.090979 13566 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.091394 13566 raft_consensus.cc:374] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.091522 13566 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.091878 13566 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.096222 13566 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.006s user 0.003s sys 0.000s
I20240502 14:17:18.097172 13572 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:18.097729 13566 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:18.100513 13566 tablet_bootstrap.cc:654] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.100872 13572 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.026s user 0.008s sys 0.009s
I20240502 14:17:18.101982 13572 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.102363 13572 raft_consensus.cc:374] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.102465 13572 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.102861 13573 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:18.103091 13573 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.018s user 0.004s sys 0.000s
I20240502 14:17:18.103730 13578 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.055s user 0.026s sys 0.018s
I20240502 14:17:18.104136 13573 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.104306 13578 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:18.104409 13573 raft_consensus.cc:374] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.104523 13573 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.104833 13573 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.105507 13573 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:17:18.105938 13573 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:18.106530 13578 tablet_bootstrap.cc:654] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.107616 13573 tablet_bootstrap.cc:654] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.108786 13584 raft_consensus.cc:482] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:18.109560 13568 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:18.109139 13584 raft_consensus.cc:504] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.109758 13568 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.043s user 0.023s sys 0.011s
I20240502 14:17:18.110565 13568 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.111003 13568 raft_consensus.cc:374] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.111089 13568 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.102788 13572 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.111364 13568 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.112394 13579 raft_consensus.cc:482] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:18.112668 13579 raft_consensus.cc:504] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.117483 13579 leader_election.cc:288] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:18.124251 13572 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.023s user 0.002s sys 0.000s
W20240502 14:17:18.124868 12618 tablet.cc:2369] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:17:18.126503 13584 leader_election.cc:288] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:18.127146 13594 raft_consensus.cc:482] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:18.127867 13594 raft_consensus.cc:504] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.136024 13568 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.026s user 0.002s sys 0.000s
I20240502 14:17:18.141341 13573 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:18.141600 13573 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.036s user 0.004s sys 0.000s
I20240502 14:17:18.142592 13573 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.143013 13573 raft_consensus.cc:374] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.143105 13573 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.143430 13573 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.151108 13573 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240502 14:17:18.166687 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:18.168346 13578 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
W20240502 14:17:18.168828 12506 leader_election.cc:341] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:18.169207 13578 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.065s user 0.026s sys 0.037s
I20240502 14:17:18.173086 13578 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.173442 13578 raft_consensus.cc:374] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.173590 13578 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
W20240502 14:17:18.173852 12355 tablet.cc:2369] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 14:17:18.175441 12487 tablet.cc:2369] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:17:18.178956 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:18.173885 13578 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.181419 13096 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:18.190436 13594 leader_election.cc:288] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069)
W20240502 14:17:18.191412 13142 tablet.cc:2369] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:17:18.206573 13578 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.037s user 0.008s sys 0.004s
I20240502 14:17:18.206925 13578 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:18.208878 13578 tablet_bootstrap.cc:654] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:18.224200 13583 raft_consensus.cc:482] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:18.224524 13583 raft_consensus.cc:504] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.225167 13566 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:18.225478 13566 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.128s user 0.006s sys 0.004s
I20240502 14:17:18.227229 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:18.227805 12964 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.227253 12965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:18.228570 12965 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:18.229856 13583 leader_election.cc:288] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:18.231643 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:18.232183 12834 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
W20240502 14:17:18.232928 13011 tablet.cc:2369] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 14:17:18.232862 13566 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.233294 13566 raft_consensus.cc:374] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.233378 13566 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.233747 13566 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.234643 13566 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.005s user 0.001s sys 0.002s
I20240502 14:17:18.235707 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:18.236104 12441 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:18.237653 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:18.238175 12309 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:18.239010 13031 leader_election.cc:302] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:18.239655 13618 raft_consensus.cc:2781] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:18.239892 13618 raft_consensus.cc:482] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:18.240121 13618 raft_consensus.cc:3037] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.240240 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:18.240446 12309 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:18.244413 13618 raft_consensus.cc:504] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.249665 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:18.249684 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:18.249904 12441 raft_consensus.cc:3037] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.250165 12703 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:18.251427 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:18.251821 12572 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:18.252501 12767 leader_election.cc:302] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters:
I20240502 14:17:18.252992 13579 raft_consensus.cc:2781] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:18.253191 13579 raft_consensus.cc:482] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:18.253304 13579 raft_consensus.cc:3037] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.254853 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:18.255086 12703 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.256394 13579 raft_consensus.cc:504] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.258179 12441 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:18.260257 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:18.260639 13096 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:18.261420 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:18.261653 12309 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.261618 12965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:18.261672 13579 leader_election.cc:288] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 election: Requested vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:18.261832 12965 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:18.262507 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:18.262722 12965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:18.262710 12309 raft_consensus.cc:3037] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.262892 12965 raft_consensus.cc:3037] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.263671 12507 leader_election.cc:302] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, d693e089cfa54fbb8df91939410b19db; no voters: eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:18.265668 13584 raft_consensus.cc:2781] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:18.265827 13584 raft_consensus.cc:482] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:18.265952 13584 raft_consensus.cc:3037] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.267001 12309 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:18.269124 13584 raft_consensus.cc:504] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.269924 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:18.270098 12572 raft_consensus.cc:3037] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.272145 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:18.272328 13096 raft_consensus.cc:3037] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.272835 12572 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:18.275362 13096 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:18.277734 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:18.277971 12703 raft_consensus.cc:3037] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.278894 12767 leader_election.cc:302] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters:
I20240502 14:17:18.279237 13579 raft_consensus.cc:2781] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:18.279954 12965 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:18.281001 12703 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:18.281459 13584 leader_election.cc:288] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Requested vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:18.281601 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:18.282006 12441 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:18.282169 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:18.282151 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:18.282348 12309 raft_consensus.cc:3037] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.282569 12572 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.283264 12965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:18.283560 12965 raft_consensus.cc:3037] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.285074 12309 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.285938 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:18.286209 13096 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:18.286984 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:18.287171 12703 raft_consensus.cc:3037] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.290270 12703 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.290872 12965 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.291383 12506 leader_election.cc:302] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, d693e089cfa54fbb8df91939410b19db; no voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:18.295482 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:18.295759 12309 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.296046 13584 raft_consensus.cc:2781] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:18.298462 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:18.298686 12834 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.299152 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:18.299503 12833 raft_consensus.cc:3037] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.302697 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:18.303165 13584 raft_consensus.cc:686] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Becoming Leader. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Running, Role: LEADER
I20240502 14:17:18.303987 13584 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.304927 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:18.305177 12703 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.306236 12833 raft_consensus.cc:2454] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.310503 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:18.310948 12833 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.311717 12965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:18.312461 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:18.312867 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:18.312961 13618 leader_election.cc:288] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Requested vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069)
I20240502 14:17:18.313227 12964 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.313809 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:18.314085 12309 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.314199 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:18.314600 12703 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.315264 13578 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:18.315218 13031 leader_election.cc:302] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, d693e089cfa54fbb8df91939410b19db
I20240502 14:17:18.315445 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:18.315569 13578 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.109s user 0.042s sys 0.030s
I20240502 14:17:18.315690 12834 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.315829 13618 raft_consensus.cc:2726] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 14:17:18.316184 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:18.316313 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:18.316407 12965 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.316478 13096 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:18.316533 12441 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:18.316365 13578 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.316704 13578 raft_consensus.cc:374] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:18.316773 13578 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:18.316828 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b33c5e313e784b97987eb07b04f615f8" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:18.317090 12440 raft_consensus.cc:3037] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.317041 13578 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.318073 13578 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:17:18.320575 12440 raft_consensus.cc:2454] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:18.321121 13579 raft_consensus.cc:686] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Becoming Leader. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Running, Role: LEADER
I20240502 14:17:18.321710 13579 consensus_queue.cc:237] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:18.322805 12636 leader_election.cc:302] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: af704d4ab2f2496a833c692bd3d25fde; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, d693e089cfa54fbb8df91939410b19db
I20240502 14:17:18.323285 13583 raft_consensus.cc:2726] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:18.324306 12166 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: term changed from 0 to 1, leader changed from <none> to 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:18.325114 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cd382683ec2b4600b6edaced79698b7e" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:18.325397 12440 raft_consensus.cc:2379] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:18.326274 13587 raft_consensus.cc:482] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:18.326462 13587 raft_consensus.cc:504] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.328632 13587 leader_election.cc:288] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:18.329041 12168 catalog_manager.cc:5484] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: term changed from 0 to 1, leader changed from <none> to c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133). New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:18.335803 13584 raft_consensus.cc:482] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:18.336020 13584 raft_consensus.cc:504] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.337276 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:18.337545 12309 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.339056 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:18.339284 12703 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.339219 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:18.339478 12833 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.339576 12965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:18.339823 12965 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.339995 13584 leader_election.cc:288] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:18.340236 12506 leader_election.cc:302] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:18.340391 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:18.340698 13096 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:18.340759 13584 raft_consensus.cc:2781] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:18.340977 13584 raft_consensus.cc:482] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:18.341146 13584 raft_consensus.cc:3037] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.344992 13584 raft_consensus.cc:504] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.346676 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:18.346932 12309 raft_consensus.cc:3037] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.350646 12309 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.352847 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:18.353129 12703 raft_consensus.cc:3037] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.356766 12703 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.357703 12965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:18.358068 12965 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:18.359701 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:18.360003 12833 raft_consensus.cc:3037] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.360338 12965 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:18.360643 12965 raft_consensus.cc:3037] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.364234 12833 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.364601 12965 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.364809 12508 leader_election.cc:302] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters:
I20240502 14:17:18.365259 13629 raft_consensus.cc:2781] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:18.365922 13584 leader_election.cc:288] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Requested vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:18.366295 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:18.366611 13096 raft_consensus.cc:3037] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.369961 13096 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.370946 13629 raft_consensus.cc:686] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Becoming Leader. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Running, Role: LEADER
I20240502 14:17:18.371508 13629 consensus_queue.cc:237] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:18.394127 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:18.400067 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:18.400426 12833 raft_consensus.cc:2379] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.402499 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:18.402741 12441 raft_consensus.cc:3037] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:18.402741 12168 catalog_manager.cc:5484] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: term changed from 0 to 1, leader changed from <none> to 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:18.403148 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:18.403417 12309 raft_consensus.cc:2379] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.404212 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:18.405475 12441 raft_consensus.cc:2454] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:18.405795 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:18.406113 12703 raft_consensus.cc:2379] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.406759 12376 leader_election.cc:302] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f
I20240502 14:17:18.407248 13587 raft_consensus.cc:2726] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:18.409746 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00aebb2064324c3eac8145d1c455622c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:18.409974 13096 raft_consensus.cc:2379] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:18.723978 13579 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.747478 13579 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.758402 13579 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.769665 13579 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.770238 13579 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.773409 13641 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 14:17:18.775739 13579 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.776171 13584 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.830365 13584 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.843338 13584 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.862615 13584 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.863700 13641 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.871235 13641 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.915549 13584 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.916002 13629 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.922549 13584 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.946283 13641 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:18.952843 13629 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:18.965775 13643 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:18.966117 13643 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:18.974758 13643 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
INSERT report
rows total: 2000
time total: 393.827 ms
time per row: 0.196913 ms
I20240502 14:17:19.526046 12163 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60530:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:19.526836 12163 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_1 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 14:17:19.536190 12287 tablet_service.cc:1467] Processing CreateTablet for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.536304 12420 tablet_service.cc:1467] Processing CreateTablet for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.536708 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76ce6bc11a2e4d12bb781b6c43c8b498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.536736 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76ce6bc11a2e4d12bb781b6c43c8b498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.536584 12551 tablet_service.cc:1467] Processing CreateTablet for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.537068 12551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76ce6bc11a2e4d12bb781b6c43c8b498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.537423 12683 tablet_service.cc:1467] Processing CreateTablet for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.537918 12683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76ce6bc11a2e4d12bb781b6c43c8b498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.542428 12814 tablet_service.cc:1467] Processing CreateTablet for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.542836 12814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76ce6bc11a2e4d12bb781b6c43c8b498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.547456 13754 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:19.548106 13755 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:19.549374 12945 tablet_service.cc:1467] Processing CreateTablet for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.549568 13076 tablet_service.cc:1467] Processing CreateTablet for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.549998 13076 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76ce6bc11a2e4d12bb781b6c43c8b498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.549997 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76ce6bc11a2e4d12bb781b6c43c8b498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.550837 13754 tablet_bootstrap.cc:654] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.550959 13755 tablet_bootstrap.cc:654] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.555652 12421 tablet_service.cc:1467] Processing CreateTablet for tablet 8b286a4283e9472980c6354935f99080 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.556262 12421 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.560407 13758 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:19.562304 13758 tablet_bootstrap.cc:654] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.563112 12287 tablet_service.cc:1467] Processing CreateTablet for tablet 8b286a4283e9472980c6354935f99080 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.563599 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.567179 12551 tablet_service.cc:1467] Processing CreateTablet for tablet 8b286a4283e9472980c6354935f99080 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.567692 12551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.567564 12682 tablet_service.cc:1467] Processing CreateTablet for tablet 8b286a4283e9472980c6354935f99080 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.568115 12682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.569962 12812 tablet_service.cc:1467] Processing CreateTablet for tablet 8b286a4283e9472980c6354935f99080 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.570394 12812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.573856 13074 tablet_service.cc:1467] Processing CreateTablet for tablet 8b286a4283e9472980c6354935f99080 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.574398 13074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.575412 13075 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.575825 13075 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.578800 12813 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.579339 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.580515 12289 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.581025 12289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.582577 12681 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.583106 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.589704 12945 tablet_service.cc:1467] Processing CreateTablet for tablet 8b286a4283e9472980c6354935f99080 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.590234 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.590063 12944 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.590763 12944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.600741 13760 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:19.602658 13760 tablet_bootstrap.cc:654] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.603576 13753 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:19.606226 12552 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.606828 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.608559 13754 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:19.608749 13754 ts_tablet_manager.cc:1367] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.061s user 0.003s sys 0.020s
I20240502 14:17:19.608796 13753 tablet_bootstrap.cc:654] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.609642 13754 raft_consensus.cc:348] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.609956 13754 raft_consensus.cc:374] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.610046 13754 raft_consensus.cc:729] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.610280 13754 consensus_queue.cc:260] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.616425 13753 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:19.616600 13753 ts_tablet_manager.cc:1367] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.000s
I20240502 14:17:19.619076 13753 raft_consensus.cc:348] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.619763 13754 ts_tablet_manager.cc:1398] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.011s user 0.007s sys 0.000s
I20240502 14:17:19.620059 13754 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:19.621793 13754 tablet_bootstrap.cc:654] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.619379 13753 raft_consensus.cc:374] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.622062 13753 raft_consensus.cc:729] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.622900 13753 consensus_queue.cc:260] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.633986 13759 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:19.635748 13759 tablet_bootstrap.cc:654] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.636947 12421 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 (DEFAULT_TABLE table=rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:19.637635 12421 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:19.637928 13754 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:19.638132 13754 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.018s user 0.004s sys 0.003s
I20240502 14:17:19.638994 13754 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.639314 13754 raft_consensus.cc:374] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.639415 13754 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.639704 13754 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.640456 13754 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:19.639936 13762 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:19.640821 13754 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:19.642421 13754 tablet_bootstrap.cc:654] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.647521 13762 tablet_bootstrap.cc:654] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.649159 13759 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:19.649327 13759 ts_tablet_manager.cc:1367] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.015s user 0.000s sys 0.007s
I20240502 14:17:19.650264 13759 raft_consensus.cc:348] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.650561 13759 raft_consensus.cc:374] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.650672 13759 raft_consensus.cc:729] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.650962 13759 consensus_queue.cc:260] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.654407 13755 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:19.654616 13755 ts_tablet_manager.cc:1367] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.107s user 0.006s sys 0.000s
I20240502 14:17:19.654707 13758 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:19.654886 13758 ts_tablet_manager.cc:1367] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.095s user 0.005s sys 0.000s
I20240502 14:17:19.655562 13755 raft_consensus.cc:348] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.655953 13755 raft_consensus.cc:374] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.656062 13755 raft_consensus.cc:729] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.655848 13758 raft_consensus.cc:348] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.656194 13758 raft_consensus.cc:374] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.656307 13758 raft_consensus.cc:729] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.656340 13755 consensus_queue.cc:260] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.656859 13758 consensus_queue.cc:260] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.658126 13755 ts_tablet_manager.cc:1398] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:19.658496 13755 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:19.658632 13754 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:19.658833 13754 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.003s
I20240502 14:17:19.659740 13754 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.660085 13754 raft_consensus.cc:374] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.660197 13754 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.660503 13755 tablet_bootstrap.cc:654] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.660560 13754 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.661800 13754 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.003s user 0.002s sys 0.002s
I20240502 14:17:19.669862 13762 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:19.670053 13758 ts_tablet_manager.cc:1398] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.015s user 0.004s sys 0.003s
I20240502 14:17:19.670203 13762 ts_tablet_manager.cc:1367] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.030s user 0.010s sys 0.000s
I20240502 14:17:19.670959 13758 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:19.671267 13762 raft_consensus.cc:348] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.671685 13762 raft_consensus.cc:374] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.671835 13762 raft_consensus.cc:729] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.672214 13762 consensus_queue.cc:260] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.672685 13758 tablet_bootstrap.cc:654] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.673247 13762 ts_tablet_manager.cc:1398] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:19.675854 13762 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:19.678109 13758 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:19.678313 13758 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.003s
I20240502 14:17:19.679186 13758 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.679487 13758 raft_consensus.cc:374] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.679563 13758 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.679844 13758 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.678150 13762 tablet_bootstrap.cc:654] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.680568 13758 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:19.684036 13758 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:19.685801 13758 tablet_bootstrap.cc:654] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.688887 13755 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:19.689110 13755 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.031s user 0.004s sys 0.000s
I20240502 14:17:19.689914 13755 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.690258 13755 raft_consensus.cc:374] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.690351 13755 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.690625 13755 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.691337 13758 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:19.691521 13758 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240502 14:17:19.692456 13758 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.692788 13758 raft_consensus.cc:374] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.692888 13758 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.694574 13755 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.005s user 0.003s sys 0.000s
I20240502 14:17:19.695276 13758 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.696128 13758 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:17:19.699326 13745 raft_consensus.cc:482] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:19.699533 13745 raft_consensus.cc:504] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.701090 13745 leader_election.cc:288] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:19.702168 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:19.702440 12309 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:19.703042 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:19.703918 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:19.704711 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
W20240502 14:17:19.705998 12508 leader_election.cc:341] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069): Illegal state: must be running to vote when last-logged opid is not known
W20240502 14:17:19.706456 12507 leader_election.cc:341] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:19.707775 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:19.708405 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:19.708665 12964 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
W20240502 14:17:19.709358 12507 leader_election.cc:341] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Illegal state: must be running to vote when last-logged opid is not known
W20240502 14:17:19.709679 12506 leader_election.cc:341] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:19.709822 12506 leader_election.cc:302] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367; no voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:19.711258 13745 raft_consensus.cc:2726] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:19.711814 13755 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:19.715983 13755 tablet_bootstrap.cc:654] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.722293 13760 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:19.723243 13759 ts_tablet_manager.cc:1398] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.074s user 0.027s sys 0.019s
I20240502 14:17:19.724179 13762 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:19.724385 13762 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.049s user 0.008s sys 0.010s
I20240502 14:17:19.722478 13760 ts_tablet_manager.cc:1367] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.122s user 0.015s sys 0.034s
I20240502 14:17:19.725400 13762 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.725538 13760 raft_consensus.cc:348] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.725726 13762 raft_consensus.cc:374] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.725798 13762 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.725811 13760 raft_consensus.cc:374] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.725912 13760 raft_consensus.cc:729] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.726038 13762 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.726644 13762 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:19.727010 13762 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:19.726208 13760 consensus_queue.cc:260] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.727633 13753 ts_tablet_manager.cc:1398] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.111s user 0.023s sys 0.021s
I20240502 14:17:19.727794 13755 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:19.728549 13762 tablet_bootstrap.cc:654] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.728873 13755 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.000s
I20240502 14:17:19.729271 13753 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:19.731315 13753 tablet_bootstrap.cc:654] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.731316 13755 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.731631 13755 raft_consensus.cc:374] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.731745 13755 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.732012 13755 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.735596 13755 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 14:17:19.737944 13759 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:19.738005 13765 raft_consensus.cc:482] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:19.738210 13765 raft_consensus.cc:504] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.740944 13760 ts_tablet_manager.cc:1398] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.016s user 0.003s sys 0.002s
I20240502 14:17:19.741019 13765 leader_election.cc:288] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:19.741668 13760 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
W20240502 14:17:19.741700 12878 acceptor_pool.cc:285] unable to collect diagnostics on RPC socket 127.27.220.133:45069: Invalid argument: socket address is ambiguous
I20240502 14:17:19.742587 13759 tablet_bootstrap.cc:654] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.743417 13760 tablet_bootstrap.cc:654] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.745612 13753 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:19.745834 13753 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.017s user 0.006s sys 0.000s
I20240502 14:17:19.746738 13753 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.747444 13753 raft_consensus.cc:374] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.747541 13753 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.747890 13753 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.748579 13753 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:19.750365 13753 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:19.752346 13753 tablet_bootstrap.cc:654] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.754846 13760 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:19.755054 13760 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.000s
I20240502 14:17:19.756456 13760 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.756821 13760 raft_consensus.cc:374] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.756918 13760 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.757264 13760 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.758061 13760 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240502 14:17:19.758431 13760 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:19.760352 13760 tablet_bootstrap.cc:654] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.761752 13762 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:19.762579 13753 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:19.762770 13753 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.000s
I20240502 14:17:19.763686 13753 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.764056 13753 raft_consensus.cc:374] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.764168 13753 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.763971 13762 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.037s user 0.009s sys 0.003s
I20240502 14:17:19.764456 13753 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.765869 13762 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.768843 13762 raft_consensus.cc:374] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.769364 13762 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.770094 13762 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.771812 13762 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 14:17:19.774075 13753 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.011s user 0.003s sys 0.002s
I20240502 14:17:19.774405 13760 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:19.774592 13760 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.016s user 0.001s sys 0.005s
I20240502 14:17:19.775296 13760 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.775626 13760 raft_consensus.cc:374] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.775765 13760 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.776048 13760 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.776636 13760 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 14:17:19.809469 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:19.809809 12833 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:19.821417 13745 raft_consensus.cc:482] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:19.821638 13745 raft_consensus.cc:504] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.822831 13759 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:19.822930 13745 leader_election.cc:288] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:19.823062 13759 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.085s user 0.013s sys 0.034s
I20240502 14:17:19.824162 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:19.824040 13759 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.824395 12703 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:19.824381 13759 raft_consensus.cc:374] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.824595 13759 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.824700 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:19.824949 13096 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:19.825007 13759 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.826228 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:19.826426 13759 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:19.826445 12833 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:19.826808 13759 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:19.826871 12508 leader_election.cc:302] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters:
I20240502 14:17:19.827143 13745 raft_consensus.cc:2781] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:19.827242 13745 raft_consensus.cc:482] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:19.827318 13745 raft_consensus.cc:3037] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.829084 13759 tablet_bootstrap.cc:654] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:19.829723 13793 raft_consensus.cc:482] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:19.829905 13793 raft_consensus.cc:504] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.831414 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:19.831983 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:19.832274 12833 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:19.833211 13793 leader_election.cc:288] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:19.833349 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:19.833633 12309 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:19.834357 13745 raft_consensus.cc:504] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.834854 12572 raft_consensus.cc:2379] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:19.836011 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:19.836210 12309 raft_consensus.cc:3037] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.836340 13745 leader_election.cc:288] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Requested vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:19.837327 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:19.837569 12833 raft_consensus.cc:3037] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.838847 12309 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:19.839998 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:19.840234 13096 raft_consensus.cc:3037] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.840353 12833 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:19.841599 13759 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:19.841791 13759 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.015s user 0.006s sys 0.005s
I20240502 14:17:19.842715 13759 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.843093 13759 raft_consensus.cc:374] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:19.843132 13096 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:19.843225 13759 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:19.843530 13759 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.843931 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:19.844163 12703 raft_consensus.cc:3037] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.844504 13759 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:19.846794 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:19.846853 12703 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:19.847015 13096 raft_consensus.cc:2379] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 70a425a77fe54cbcb9755f9c540fd367 in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:19.847221 12507 leader_election.cc:302] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters:
I20240502 14:17:19.847682 13745 raft_consensus.cc:2781] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:19.848469 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:19.848702 12309 raft_consensus.cc:2379] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:19.849169 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:19.848999 13745 raft_consensus.cc:686] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Becoming Leader. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Running, Role: LEADER
I20240502 14:17:19.849699 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:19.849948 13096 raft_consensus.cc:2379] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:19.849496 12703 raft_consensus.cc:2379] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:19.850121 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:19.850314 12440 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:19.850610 13745 consensus_queue.cc:237] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.853236 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:19.853440 12441 raft_consensus.cc:3037] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.853585 12376 leader_election.cc:302] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:19.855186 12163 catalog_manager.cc:5484] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: term changed from 0 to 1, leader changed from <none> to 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:19.856076 12441 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:19.857887 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:19.860813 13793 raft_consensus.cc:2726] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:19.861109 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:19.861398 12703 raft_consensus.cc:2379] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 70a425a77fe54cbcb9755f9c540fd367 in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:19.861995 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:19.862210 12441 raft_consensus.cc:2379] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 70a425a77fe54cbcb9755f9c540fd367 in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:19.863247 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:19.863332 12963 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:19.863533 12964 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:19.869720 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:19.869954 12964 raft_consensus.cc:3037] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.873397 12964 raft_consensus.cc:2454] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:19.879009 13771 raft_consensus.cc:482] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:19.879191 13771 raft_consensus.cc:504] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.880540 13771 leader_election.cc:288] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:19.880947 12244 leader_election.cc:302] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:19.881582 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:19.881597 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:19.881810 12572 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:19.881824 12833 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:19.882283 13765 raft_consensus.cc:2726] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:19.882432 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:19.882719 12441 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:19.882897 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:19.883138 12703 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:19.883188 13030 leader_election.cc:302] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:19.883635 13771 raft_consensus.cc:2781] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:19.883755 13771 raft_consensus.cc:482] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:19.883849 13771 raft_consensus.cc:3037] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.886461 13771 raft_consensus.cc:504] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.887529 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:19.887744 12441 raft_consensus.cc:3037] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.887799 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:19.887974 12572 raft_consensus.cc:3037] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.890549 12572 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:19.891073 12441 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:19.891769 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:19.891774 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:19.891969 12833 raft_consensus.cc:3037] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.891973 12703 raft_consensus.cc:3037] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.894973 12703 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:19.895411 12833 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:19.896363 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:19.896616 12964 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:19.897562 13771 leader_election.cc:288] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Requested vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:19.898499 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:19.898738 12309 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:19.899706 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:19.899893 12309 raft_consensus.cc:3037] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.900775 12963 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8b286a4283e9472980c6354935f99080" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:19.900982 12963 raft_consensus.cc:3037] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.903187 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "76ce6bc11a2e4d12bb781b6c43c8b498" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:19.903439 12964 raft_consensus.cc:2379] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 70a425a77fe54cbcb9755f9c540fd367 in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:19.903898 12963 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:19.904194 13032 leader_election.cc:302] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters:
I20240502 14:17:19.904520 13771 raft_consensus.cc:2781] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:19.905426 12309 raft_consensus.cc:2454] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:19.911248 13628 raft_consensus.cc:482] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:19.911478 13628 raft_consensus.cc:504] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.913168 13628 leader_election.cc:288] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:19.914904 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:19.915113 13096 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:19.915827 13771 raft_consensus.cc:686] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Becoming Leader. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Running, Role: LEADER
I20240502 14:17:19.916239 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:19.916507 12572 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:19.916426 13771 consensus_queue.cc:237] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.917261 12963 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:19.918257 12963 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:19.918650 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:19.919234 12768 leader_election.cc:302] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:19.919605 13628 raft_consensus.cc:2781] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:19.919708 13628 raft_consensus.cc:482] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:19.919800 13628 raft_consensus.cc:3037] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.922808 13628 raft_consensus.cc:504] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.924713 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:19.925015 12309 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:19.925675 12703 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:19.926074 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:19.926247 12309 raft_consensus.cc:3037] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.930925 12309 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:19.933210 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:19.933707 12572 raft_consensus.cc:3037] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.938552 12963 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:19.938923 12963 raft_consensus.cc:3037] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.942595 12963 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:19.943701 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:19.944062 12703 raft_consensus.cc:3037] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.946797 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:19.947103 12440 raft_consensus.cc:3037] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.950999 12440 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:19.952281 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:19.952574 13096 raft_consensus.cc:3037] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:19.956346 13096 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:19.959512 12703 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:19.963021 13628 leader_election.cc:288] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 election: Requested vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:19.963634 12768 leader_election.cc:302] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:19.964285 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae8e0c47ce54724899b56aeb1b31336" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:19.964764 12441 raft_consensus.cc:2362] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1. Re-sending same reply.
I20240502 14:17:19.966147 13628 raft_consensus.cc:2781] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:19.967137 12572 raft_consensus.cc:2454] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 1.
I20240502 14:17:19.971237 13628 raft_consensus.cc:686] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Becoming Leader. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Running, Role: LEADER
I20240502 14:17:19.971803 13628 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:19.985194 12163 catalog_manager.cc:5484] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: term changed from 0 to 1, leader changed from <none> to 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135). New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:19.985321 12167 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: term changed from 0 to 1, leader changed from <none> to c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133). New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:20.242218 13745 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.242604 13723 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.253484 13723 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.259402 13723 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.275841 13811 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 14:17:20.276706 13813 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.368794 13771 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.399682 13771 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.405931 13628 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.426702 13771 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.428656 13628 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.434173 13628 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.466568 13628 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.467483 13771 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.492755 13835 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.494027 13837 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:20.503120 13836 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:20.503698 13805 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:20.504303 13805 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:17:20.507715 13836 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:20.519274 13805 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
INSERT report
rows total: 2000
time total: 462.511 ms
time per row: 0.231256 ms
I20240502 14:17:21.130605 12165 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60530:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:21.131448 12165 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_2 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 14:17:21.141083 12289 tablet_service.cc:1467] Processing CreateTablet for tablet 3b3082d181004353bdff4414cd264d7c (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.141326 12421 tablet_service.cc:1467] Processing CreateTablet for tablet 3b3082d181004353bdff4414cd264d7c (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.141606 12289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.141991 12681 tablet_service.cc:1467] Processing CreateTablet for tablet 3b3082d181004353bdff4414cd264d7c (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.142311 12945 tablet_service.cc:1467] Processing CreateTablet for tablet 3b3082d181004353bdff4414cd264d7c (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.142443 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.142740 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.142948 13076 tablet_service.cc:1467] Processing CreateTablet for tablet 3b3082d181004353bdff4414cd264d7c (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.143399 13076 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.148291 12421 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.148229 12551 tablet_service.cc:1467] Processing CreateTablet for tablet 3b3082d181004353bdff4414cd264d7c (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.148653 12551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.154028 12813 tablet_service.cc:1467] Processing CreateTablet for tablet 3b3082d181004353bdff4414cd264d7c (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.154139 13911 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:21.154560 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.157404 13911 tablet_bootstrap.cc:654] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.162658 12289 tablet_service.cc:1467] Processing CreateTablet for tablet df48755615874edab65577e92249fc78 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.163075 12289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.163590 12420 tablet_service.cc:1467] Processing CreateTablet for tablet df48755615874edab65577e92249fc78 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.164139 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.164518 12552 tablet_service.cc:1467] Processing CreateTablet for tablet df48755615874edab65577e92249fc78 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.164922 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.166388 12682 tablet_service.cc:1467] Processing CreateTablet for tablet df48755615874edab65577e92249fc78 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.166796 12682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.168079 13913 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:21.169839 13913 tablet_bootstrap.cc:654] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.170756 12812 tablet_service.cc:1467] Processing CreateTablet for tablet df48755615874edab65577e92249fc78 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.171504 13914 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:21.171634 12812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.171885 12944 tablet_service.cc:1467] Processing CreateTablet for tablet df48755615874edab65577e92249fc78 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.172344 12944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.172467 13074 tablet_service.cc:1467] Processing CreateTablet for tablet df48755615874edab65577e92249fc78 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.172904 13074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.174352 13914 tablet_bootstrap.cc:654] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.181653 13918 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:21.185324 12814 tablet_service.cc:1467] Processing CreateTablet for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.185498 12944 tablet_service.cc:1467] Processing CreateTablet for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.185763 12814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c4323e7ed9d4784ba7f8dd331036fb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.185879 12944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c4323e7ed9d4784ba7f8dd331036fb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.194099 12287 tablet_service.cc:1467] Processing CreateTablet for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.194430 12420 tablet_service.cc:1467] Processing CreateTablet for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.194567 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c4323e7ed9d4784ba7f8dd331036fb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.194844 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c4323e7ed9d4784ba7f8dd331036fb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.194844 13076 tablet_service.cc:1467] Processing CreateTablet for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.195250 13076 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c4323e7ed9d4784ba7f8dd331036fb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.198618 12550 tablet_service.cc:1467] Processing CreateTablet for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.199056 12550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c4323e7ed9d4784ba7f8dd331036fb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.206735 13918 tablet_bootstrap.cc:654] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.208611 13919 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:21.213428 13917 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:21.215503 13914 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:21.215849 13914 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.049s user 0.004s sys 0.015s
I20240502 14:17:21.217849 13917 tablet_bootstrap.cc:654] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.217896 13912 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:21.217756 13914 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.218062 13918 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:21.218050 13914 raft_consensus.cc:374] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.218154 13914 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.218235 13918 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.037s user 0.007s sys 0.000s
I20240502 14:17:21.218446 13914 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.219087 13918 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.219507 13911 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:21.219513 13918 raft_consensus.cc:374] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.219621 13918 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.219947 13918 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.220562 13911 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.067s user 0.005s sys 0.006s
I20240502 14:17:21.221438 13911 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.221727 13911 raft_consensus.cc:374] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.221825 13911 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.222112 13918 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 14:17:21.222064 13911 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.222512 13918 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:21.224457 13918 tablet_bootstrap.cc:654] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.225046 13912 tablet_bootstrap.cc:654] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.226390 13911 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 14:17:21.226682 13911 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:21.226920 12681 tablet_service.cc:1467] Processing CreateTablet for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 (DEFAULT_TABLE table=rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:21.228065 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c4323e7ed9d4784ba7f8dd331036fb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:21.228286 13911 tablet_bootstrap.cc:654] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.234313 13912 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:21.234421 13914 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.018s user 0.010s sys 0.000s
I20240502 14:17:21.234500 13912 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.000s
I20240502 14:17:21.234735 13914 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:21.235288 13912 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.235584 13912 raft_consensus.cc:374] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.235678 13912 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.235946 13912 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.236424 13914 tablet_bootstrap.cc:654] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.239596 13919 tablet_bootstrap.cc:654] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.243021 13918 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:21.243268 13918 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.021s user 0.001s sys 0.004s
I20240502 14:17:21.244146 13913 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:21.244405 13913 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.076s user 0.004s sys 0.000s
I20240502 14:17:21.244293 13918 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.244765 13918 raft_consensus.cc:374] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.244868 13918 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.245241 13918 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.246143 13918 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:21.246364 13912 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.011s user 0.002s sys 0.004s
I20240502 14:17:21.246486 13918 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:21.246675 13912 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:21.250365 13912 tablet_bootstrap.cc:654] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.254101 13914 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:21.254276 13914 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.020s user 0.009s sys 0.009s
I20240502 14:17:21.254946 13914 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.255400 13914 raft_consensus.cc:374] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.255481 13914 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.255836 13918 tablet_bootstrap.cc:654] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.255779 13914 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.246279 13913 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.256440 13914 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:21.256880 13914 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:21.257112 13913 raft_consensus.cc:374] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.257640 13913 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.258749 13914 tablet_bootstrap.cc:654] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.258234 13913 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.259775 13926 raft_consensus.cc:482] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:21.259964 13926 raft_consensus.cc:504] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.261255 13919 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:21.262343 13919 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.054s user 0.005s sys 0.000s
I20240502 14:17:21.270462 13914 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:21.270637 13914 ts_tablet_manager.cc:1367] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.014s user 0.006s sys 0.003s
I20240502 14:17:21.271596 13911 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:21.272099 13918 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:21.272264 13918 ts_tablet_manager.cc:1367] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.026s user 0.003s sys 0.002s
I20240502 14:17:21.273891 13917 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:21.273224 13918 raft_consensus.cc:348] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.276309 13918 raft_consensus.cc:374] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.276451 13918 raft_consensus.cc:729] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.277101 13918 consensus_queue.cc:260] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.279673 13918 ts_tablet_manager.cc:1398] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 14:17:21.282143 13917 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.069s user 0.003s sys 0.003s
I20240502 14:17:21.283164 13914 raft_consensus.cc:348] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.284163 13914 raft_consensus.cc:374] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.284292 13914 raft_consensus.cc:729] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.285218 13914 consensus_queue.cc:260] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.263971 13919 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.293880 13911 ts_tablet_manager.cc:1367] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.067s user 0.008s sys 0.007s
I20240502 14:17:21.295800 13921 raft_consensus.cc:482] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:21.295984 13921 raft_consensus.cc:504] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.297355 13921 leader_election.cc:288] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:21.297896 13914 ts_tablet_manager.cc:1398] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.015s user 0.003s sys 0.000s
I20240502 14:17:21.298713 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:21.299024 12703 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:21.300009 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
W20240502 14:17:21.301039 12376 leader_election.cc:341] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:21.303519 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:21.303798 12572 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:21.297824 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:21.305111 13919 raft_consensus.cc:374] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.305298 13919 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.297935 13922 raft_consensus.cc:482] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:21.305898 13922 raft_consensus.cc:504] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.306970 13926 leader_election.cc:288] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:21.307767 13922 leader_election.cc:288] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:21.298501 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:21.308382 12307 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:21.308221 13919 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.308670 12833 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:21.308662 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:21.308912 12572 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:21.309562 13919 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.046s user 0.002s sys 0.000s
I20240502 14:17:21.309501 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:21.309752 12440 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:21.310902 13911 raft_consensus.cc:348] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.311267 12377 leader_election.cc:302] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters: 84d1fcd2d4cf472db7959f63a7f0ddaf
I20240502 14:17:21.311529 13911 raft_consensus.cc:374] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.311605 13921 raft_consensus.cc:2781] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:21.311749 13921 raft_consensus.cc:482] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:21.311885 13921 raft_consensus.cc:3037] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.313313 13913 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.069s user 0.005s sys 0.000s
I20240502 14:17:21.313716 13913 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:21.315044 13921 raft_consensus.cc:504] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.283174 13917 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.315742 13917 raft_consensus.cc:374] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.315912 13917 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.316260 13917 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.317183 13917 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.035s user 0.000s sys 0.002s
I20240502 14:17:21.319623 13919 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:21.321731 13919 tablet_bootstrap.cc:654] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.321784 13813 raft_consensus.cc:482] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:21.321923 13911 raft_consensus.cc:729] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.317629 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:21.321995 13813 raft_consensus.cc:504] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.322304 12440 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.322394 13911 consensus_queue.cc:260] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.323174 13911 ts_tablet_manager.cc:1398] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.017s user 0.001s sys 0.001s
I20240502 14:17:21.323491 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:21.323540 13813 leader_election.cc:288] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:21.323745 12703 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:21.321475 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:21.324615 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:21.324836 12572 raft_consensus.cc:3037] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.325310 13912 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:21.325439 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:21.325673 12703 raft_consensus.cc:3037] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.326375 13917 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:21.326478 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:21.326694 12440 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:21.326025 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:21.326948 12307 raft_consensus.cc:3037] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.327751 12572 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:21.328454 12703 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:21.329183 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:21.330138 12307 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:21.330554 13912 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.084s user 0.001s sys 0.005s
I20240502 14:17:21.330832 12376 leader_election.cc:302] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:21.330036 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
W20240502 14:17:21.332185 12507 leader_election.cc:341] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
W20240502 14:17:21.332458 12507 leader_election.cc:341] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:21.332949 13911 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:21.332661 13921 leader_election.cc:288] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 election: Requested vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:21.333604 13921 raft_consensus.cc:2781] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:21.333760 13921 raft_consensus.cc:686] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Becoming Leader. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Running, Role: LEADER
I20240502 14:17:21.334116 13913 tablet_bootstrap.cc:654] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.334427 13921 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.336748 13917 tablet_bootstrap.cc:654] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.336988 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:21.337325 12572 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.344357 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:21.347059 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:21.347294 12307 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
W20240502 14:17:21.347678 12244 leader_election.cc:341] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:21.348007 12244 leader_election.cc:302] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, eddeff583cf34c8dab2942fb36abbf9e; no voters: 84d1fcd2d4cf472db7959f63a7f0ddaf
I20240502 14:17:21.348312 13922 raft_consensus.cc:2741] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election decision vote started in defunct term 0: won
I20240502 14:17:21.349298 12163 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: term changed from 0 to 1, leader changed from <none> to eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130). New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } }
W20240502 14:17:21.353716 12506 leader_election.cc:341] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:21.355947 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:21.356215 12307 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.357615 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:21.357865 12964 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:21.358426 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:21.358680 12964 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:21.324247 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:21.328316 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:21.358976 12833 raft_consensus.cc:3037] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.359165 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
W20240502 14:17:21.359285 12508 leader_election.cc:341] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:21.359328 12964 raft_consensus.cc:3037] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.359421 12508 leader_election.cc:302] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 7 responses out of 7 voters: 3 yes votes; 4 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, eddeff583cf34c8dab2942fb36abbf9e; no voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db
I20240502 14:17:21.359926 13813 raft_consensus.cc:2726] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:21.350128 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:21.362221 12964 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:21.362234 12833 raft_consensus.cc:2454] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:21.363579 13912 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.364549 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:21.364814 12833 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.364791 13912 raft_consensus.cc:374] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.365056 13912 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.365440 13912 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.366540 13911 tablet_bootstrap.cc:654] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.349490 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:21.353855 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
W20240502 14:17:21.367842 12376 leader_election.cc:341] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:21.368552 13912 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.037s user 0.002s sys 0.000s
I20240502 14:17:21.368906 13912 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:21.365648 12901 leader_election.cc:302] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: d693e089cfa54fbb8df91939410b19db; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:21.369350 13926 raft_consensus.cc:2726] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:21.370340 13912 tablet_bootstrap.cc:654] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
W20240502 14:17:21.370711 12900 leader_election.cc:341] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:21.374591 13911 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:21.374763 13911 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.044s user 0.000s sys 0.003s
I20240502 14:17:21.375672 13911 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.376044 13911 raft_consensus.cc:374] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.376142 13911 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.376442 13911 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.377215 13911 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.002s user 0.001s sys 0.002s
I20240502 14:17:21.380668 13912 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:21.380973 13912 ts_tablet_manager.cc:1367] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.002s
I20240502 14:17:21.381763 13912 raft_consensus.cc:348] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.381942 13912 raft_consensus.cc:374] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.382026 13912 raft_consensus.cc:729] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.382239 13912 consensus_queue.cc:260] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.382439 13919 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:21.382627 13919 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.063s user 0.000s sys 0.004s
I20240502 14:17:21.382762 13912 ts_tablet_manager.cc:1398] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 14:17:21.383447 13919 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.383803 13919 raft_consensus.cc:374] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.383920 13919 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.384064 13913 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:21.384312 13913 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.071s user 0.007s sys 0.000s
I20240502 14:17:21.384253 13919 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.385136 13919 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:21.385246 13913 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.385478 13919 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:21.385574 13913 raft_consensus.cc:374] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.385663 13913 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.386104 13913 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.386723 13913 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:21.387074 13913 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:21.387295 13919 tablet_bootstrap.cc:654] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.388456 13830 raft_consensus.cc:482] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:21.388687 13913 tablet_bootstrap.cc:654] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.388624 13830 raft_consensus.cc:504] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.389744 13830 leader_election.cc:288] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:21.390547 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:21.390645 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:21.390796 12703 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:21.390873 12572 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:21.391192 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:21.391287 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:21.391458 12833 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:21.391463 12307 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:21.391798 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:21.392014 12964 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:21.392509 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:21.392532 13030 leader_election.cc:302] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:21.392712 12440 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:21.393173 13830 raft_consensus.cc:2781] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:21.393317 13830 raft_consensus.cc:482] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:21.393438 13830 raft_consensus.cc:3037] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.396179 13830 raft_consensus.cc:504] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.398430 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:21.398713 12703 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.398685 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:21.398919 12307 raft_consensus.cc:3037] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.400424 13917 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:21.400622 13917 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.074s user 0.005s sys 0.002s
I20240502 14:17:21.401764 12307 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:21.402968 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:21.403210 12440 raft_consensus.cc:3037] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.403226 13917 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.403554 13917 raft_consensus.cc:374] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.403858 13917 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.404162 13917 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.405485 13917 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.005s user 0.001s sys 0.001s
I20240502 14:17:21.406205 12440 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:21.407418 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:21.407709 12572 raft_consensus.cc:3037] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.409745 13917 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:21.410632 12572 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:21.411034 13029 leader_election.cc:302] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:21.411360 13835 raft_consensus.cc:2781] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:21.411561 13835 raft_consensus.cc:686] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Becoming Leader. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Running, Role: LEADER
I20240502 14:17:21.411785 13830 leader_election.cc:288] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Requested vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:21.411945 13835 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.412284 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:21.412559 12703 raft_consensus.cc:3037] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.414265 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:21.414440 12833 raft_consensus.cc:3037] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.416033 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "df48755615874edab65577e92249fc78" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:21.416363 12964 raft_consensus.cc:3037] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.417032 12833 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:21.418912 13832 raft_consensus.cc:482] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:21.419061 13832 raft_consensus.cc:504] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:21.420032 12964 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:21.420198 13832 leader_election.cc:288] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:21.420975 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:21.422912 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:21.422940 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:21.423072 12963 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:21.423197 12701 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.423103 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:21.423224 12307 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.423334 12572 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.423350 12963 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.423830 13029 leader_election.cc:302] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:21.422801 12703 raft_consensus.cc:2454] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:21.424189 13832 raft_consensus.cc:3037] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.425125 13917 tablet_bootstrap.cc:654] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:21.426913 13832 raft_consensus.cc:2726] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:21.427237 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3b3082d181004353bdff4414cd264d7c" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:21.427551 12833 raft_consensus.cc:2379] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:21.427752 13913 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:21.427974 13913 ts_tablet_manager.cc:1367] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.041s user 0.009s sys 0.007s
I20240502 14:17:21.429037 13913 raft_consensus.cc:348] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.429447 13913 raft_consensus.cc:374] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.429605 13913 raft_consensus.cc:729] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.429958 13913 consensus_queue.cc:260] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.430903 13913 ts_tablet_manager.cc:1398] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:21.433607 13919 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:21.434165 13919 ts_tablet_manager.cc:1367] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.049s user 0.009s sys 0.010s
I20240502 14:17:21.435750 13919 raft_consensus.cc:348] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.436729 13919 raft_consensus.cc:374] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.437055 13919 raft_consensus.cc:729] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.437443 13919 consensus_queue.cc:260] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.438970 13919 ts_tablet_manager.cc:1398] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:17:21.444995 13917 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:21.445223 13917 ts_tablet_manager.cc:1367] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.036s user 0.006s sys 0.013s
I20240502 14:17:21.446147 13917 raft_consensus.cc:348] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.446485 13917 raft_consensus.cc:374] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:21.446817 13917 raft_consensus.cc:729] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:21.447083 13917 consensus_queue.cc:260] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.447717 13917 ts_tablet_manager.cc:1398] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 14:17:21.447469 12163 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: term changed from 0 to 1, leader changed from <none> to 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135). New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:21.472328 13926 raft_consensus.cc:482] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:21.472517 13926 raft_consensus.cc:504] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.473752 13926 leader_election.cc:288] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:21.473838 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:21.474102 12833 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:21.474092 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:21.474050 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:21.474298 13096 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:21.474305 12307 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:21.474515 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:21.474743 12440 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:21.474751 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:21.474843 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:21.474996 12703 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:21.475080 12572 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:21.475101 12900 leader_election.cc:302] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:21.475427 13926 raft_consensus.cc:2781] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:21.475539 13926 raft_consensus.cc:482] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:21.475634 13926 raft_consensus.cc:3037] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.477674 13926 raft_consensus.cc:504] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.478727 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:21.478924 12703 raft_consensus.cc:3037] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.478904 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:21.479027 13926 leader_election.cc:288] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Requested vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:21.479118 12833 raft_consensus.cc:3037] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.479749 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:21.479929 13096 raft_consensus.cc:3037] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.479887 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:21.480047 12440 raft_consensus.cc:3037] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.481563 12703 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:21.481714 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:21.481843 12833 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:21.481907 12307 raft_consensus.cc:3037] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.482293 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c4323e7ed9d4784ba7f8dd331036fb3" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:21.482481 12572 raft_consensus.cc:3037] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:21.482854 13096 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:21.483906 12900 leader_election.cc:302] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:21.484155 12440 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:21.484315 13926 raft_consensus.cc:2781] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:21.484690 12307 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:21.485241 12572 raft_consensus.cc:2454] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:21.485555 13926 raft_consensus.cc:686] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Becoming Leader. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Running, Role: LEADER
I20240502 14:17:21.486141 13926 consensus_queue.cc:237] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:21.490235 12163 catalog_manager.cc:5484] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db reported cstate change: term changed from 0 to 1, leader changed from <none> to d693e089cfa54fbb8df91939410b19db (127.27.220.134). New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:21.795490 13943 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.804656 13943 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.857880 13835 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.859897 13835 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:21.874298 13957 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:21.874639 13957 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:17:21.883394 13943 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.886318 13926 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:21.887116 13957 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:17:21.894241 13943 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.895102 13926 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.920902 13969 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.942711 13969 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.957530 13835 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.967921 13835 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.972733 13926 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:21.987221 13926 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:22.032089 13986 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:22.038064 13991 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:22.057206 14000 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
I20240502 14:17:22.074301 13926 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
INSERT report
rows total: 2000
time total: 513.676 ms
time per row: 0.256838 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 14:17:23.656989 12163 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60530:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:23.657778 12163 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_3 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 14:17:23.666555 12945 tablet_service.cc:1467] Processing CreateTablet for tablet c9ff45109a6d4dc9aba4617819929981 (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.667032 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.667325 13076 tablet_service.cc:1467] Processing CreateTablet for tablet c9ff45109a6d4dc9aba4617819929981 (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.667739 13076 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.667800 12420 tablet_service.cc:1467] Processing CreateTablet for tablet c9ff45109a6d4dc9aba4617819929981 (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.667822 12681 tablet_service.cc:1467] Processing CreateTablet for tablet c9ff45109a6d4dc9aba4617819929981 (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.668198 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.668215 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.672014 12552 tablet_service.cc:1467] Processing CreateTablet for tablet c9ff45109a6d4dc9aba4617819929981 (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.672055 12813 tablet_service.cc:1467] Processing CreateTablet for tablet c9ff45109a6d4dc9aba4617819929981 (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.672468 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.672468 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.676460 12289 tablet_service.cc:1467] Processing CreateTablet for tablet c9ff45109a6d4dc9aba4617819929981 (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.676878 12289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.682621 14067 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:23.683418 14065 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:23.685751 14065 tablet_bootstrap.cc:654] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.686829 14067 tablet_bootstrap.cc:654] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.687933 14066 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:23.689731 14066 tablet_bootstrap.cc:654] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.691519 12682 tablet_service.cc:1467] Processing CreateTablet for tablet 01ffac38da1d4ece9cba649a59d5518b (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.692013 12682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.691951 12945 tablet_service.cc:1467] Processing CreateTablet for tablet 01ffac38da1d4ece9cba649a59d5518b (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.692382 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.697196 12813 tablet_service.cc:1467] Processing CreateTablet for tablet 01ffac38da1d4ece9cba649a59d5518b (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.697633 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.698019 12287 tablet_service.cc:1467] Processing CreateTablet for tablet 01ffac38da1d4ece9cba649a59d5518b (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.698424 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.701928 13074 tablet_service.cc:1467] Processing CreateTablet for tablet 01ffac38da1d4ece9cba649a59d5518b (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.702378 13074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.708688 14070 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:23.709129 12420 tablet_service.cc:1467] Processing CreateTablet for tablet 01ffac38da1d4ece9cba649a59d5518b (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.709549 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.711961 14070 tablet_bootstrap.cc:654] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.714252 14065 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:23.714507 14065 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.031s user 0.003s sys 0.008s
I20240502 14:17:23.715514 14065 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.716039 14065 raft_consensus.cc:374] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.716221 14065 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.716605 14065 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.717381 14067 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:23.717564 14065 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:23.718005 14067 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.036s user 0.017s sys 0.000s
I20240502 14:17:23.718914 14070 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:23.719147 14070 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.000s
I20240502 14:17:23.719277 14066 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:23.719473 14066 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.032s user 0.001s sys 0.003s
I20240502 14:17:23.719843 14067 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.720146 14070 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.720243 14066 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.720543 14066 raft_consensus.cc:374] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.720604 14067 raft_consensus.cc:374] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.720585 14070 raft_consensus.cc:374] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.720650 14066 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.720731 14070 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.720933 14067 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.720958 14066 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.721110 14070 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.721608 14066 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.002s user 0.001s sys 0.002s
I20240502 14:17:23.721479 14067 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.723040 14067 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 14:17:23.724184 14067 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:23.726953 14067 tablet_bootstrap.cc:654] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.728443 14070 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.009s user 0.002s sys 0.006s
I20240502 14:17:23.732525 12550 tablet_service.cc:1467] Processing CreateTablet for tablet 01ffac38da1d4ece9cba649a59d5518b (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.732990 12550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.733770 14070 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:23.736632 14066 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:23.737056 14070 tablet_bootstrap.cc:654] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.739493 14066 tablet_bootstrap.cc:654] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.740943 12681 tablet_service.cc:1467] Processing CreateTablet for tablet a6d444ae07234a30be060fbcc5b63b4f (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.741475 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.743508 12813 tablet_service.cc:1467] Processing CreateTablet for tablet a6d444ae07234a30be060fbcc5b63b4f (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.743955 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.744762 13074 tablet_service.cc:1467] Processing CreateTablet for tablet a6d444ae07234a30be060fbcc5b63b4f (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.745213 13074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.745460 12288 tablet_service.cc:1467] Processing CreateTablet for tablet a6d444ae07234a30be060fbcc5b63b4f (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.746066 12288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.746132 12421 tablet_service.cc:1467] Processing CreateTablet for tablet a6d444ae07234a30be060fbcc5b63b4f (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.746590 12421 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.749598 14070 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:23.749996 14070 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.016s user 0.005s sys 0.001s
I20240502 14:17:23.750037 12945 tablet_service.cc:1467] Processing CreateTablet for tablet a6d444ae07234a30be060fbcc5b63b4f (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.751120 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.754772 14070 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.755187 14070 raft_consensus.cc:374] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.755741 14065 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:23.757921 14065 tablet_bootstrap.cc:654] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.755343 14070 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.759385 14070 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.760378 14065 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:23.760550 14065 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.001s
I20240502 14:17:23.761616 12550 tablet_service.cc:1467] Processing CreateTablet for tablet a6d444ae07234a30be060fbcc5b63b4f (DEFAULT_TABLE table=rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:23.762121 12550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:23.762104 14067 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:23.762370 14067 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.038s user 0.000s sys 0.006s
I20240502 14:17:23.763022 14070 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.013s user 0.002s sys 0.000s
I20240502 14:17:23.763274 14067 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.763614 14067 raft_consensus.cc:374] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.763701 14067 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.763978 14067 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.764672 14067 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:17:23.761281 14065 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.765053 14065 raft_consensus.cc:374] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.765151 14065 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.765421 14065 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.766170 14065 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.006s user 0.001s sys 0.001s
I20240502 14:17:23.766507 14065 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:23.768086 14065 tablet_bootstrap.cc:654] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.769944 14070 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:23.770252 14067 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:23.771896 14067 tablet_bootstrap.cc:654] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.772215 14070 tablet_bootstrap.cc:654] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.776270 14070 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:23.776491 14070 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.001s
I20240502 14:17:23.777480 14070 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.777884 14070 raft_consensus.cc:374] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.778007 14070 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.779742 14065 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:23.779950 14065 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.014s user 0.007s sys 0.002s
I20240502 14:17:23.780946 14065 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.781330 14065 raft_consensus.cc:374] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.781446 14065 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.781797 14065 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.782764 14065 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240502 14:17:23.783224 14070 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.784916 14070 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.008s user 0.003s sys 0.000s
I20240502 14:17:23.793998 13973 raft_consensus.cc:482] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:23.794225 13973 raft_consensus.cc:504] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.795848 13973 leader_election.cc:288] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:23.796794 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:23.796990 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
W20240502 14:17:23.798162 12377 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:23.798395 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
W20240502 14:17:23.798497 12376 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:23.799098 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:23.799333 12703 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:23.799510 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
W20240502 14:17:23.799995 12374 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): Not found: Tablet not found: a6d444ae07234a30be060fbcc5b63b4f
W20240502 14:17:23.800287 12375 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:23.800441 12375 leader_election.cc:302] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db
W20240502 14:17:23.800979 12376 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:23.803776 14077 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:23.804190 13973 raft_consensus.cc:2726] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:23.806022 14078 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:23.808091 14078 tablet_bootstrap.cc:654] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.814328 14077 tablet_bootstrap.cc:654] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.814903 14067 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:23.815097 14067 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.045s user 0.001s sys 0.018s
I20240502 14:17:23.815886 14067 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.816179 14067 raft_consensus.cc:374] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.816267 14067 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.816504 14067 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.817265 14067 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.002s user 0.000s sys 0.003s
I20240502 14:17:23.819382 14078 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:23.819593 14078 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.014s user 0.003s sys 0.000s
I20240502 14:17:23.820591 14078 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.820999 14078 raft_consensus.cc:374] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.821105 14078 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.822832 14066 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:23.823055 14066 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.086s user 0.007s sys 0.019s
I20240502 14:17:23.823103 14078 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.823810 14066 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.824123 14066 raft_consensus.cc:374] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.824210 14066 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.824748 14066 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.826216 14078 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.006s user 0.003s sys 0.000s
I20240502 14:17:23.827791 14066 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.005s user 0.001s sys 0.002s
I20240502 14:17:23.828433 14064 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:23.828860 14078 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:23.830138 14064 tablet_bootstrap.cc:654] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.830376 14078 tablet_bootstrap.cc:654] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.831223 14077 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:23.831416 14077 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.028s user 0.002s sys 0.002s
I20240502 14:17:23.832226 14077 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.832510 14077 raft_consensus.cc:374] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.832589 14077 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.832808 14077 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.834065 14066 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:23.835623 14049 raft_consensus.cc:482] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:23.835870 14049 raft_consensus.cc:504] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.836571 14066 tablet_bootstrap.cc:654] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.837687 14049 leader_election.cc:288] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:23.839566 13973 raft_consensus.cc:482] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:23.839751 13973 raft_consensus.cc:504] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.840121 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:23.840453 12703 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:23.840925 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:23.841261 13973 leader_election.cc:288] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:23.841562 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:23.842566 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:23.842659 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:23.842875 12833 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:23.842905 12964 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:23.843281 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:23.843724 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:23.843994 12572 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
W20240502 14:17:23.843922 12769 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
W20240502 14:17:23.844250 12769 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:23.845551 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:23.845830 12440 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c78bd31d8d2142bf9d8e06b5a0074e0f in term 0.
I20240502 14:17:23.846164 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
W20240502 14:17:23.846876 12767 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): Illegal state: must be running to vote when last-logged opid is not known
W20240502 14:17:23.847191 12768 leader_election.cc:341] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:23.847317 12768 leader_election.cc:302] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, d693e089cfa54fbb8df91939410b19db
I20240502 14:17:23.847659 14049 raft_consensus.cc:2726] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:23.848346 12375 leader_election.cc:302] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:23.848883 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:23.850149 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:23.850282 14064 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:23.850422 12309 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:23.850557 14064 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.022s user 0.005s sys 0.003s
I20240502 14:17:23.850909 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:23.851127 12703 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
W20240502 14:17:23.851469 12376 leader_election.cc:341] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:23.852056 14077 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.021s user 0.008s sys 0.013s
I20240502 14:17:23.852108 13973 raft_consensus.cc:2781] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:23.852226 13973 raft_consensus.cc:482] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:23.852325 13973 raft_consensus.cc:3037] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.852015 14064 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.852669 14064 raft_consensus.cc:374] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.853010 14064 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.853071 14077 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:23.854990 14064 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.854779 14077 tablet_bootstrap.cc:654] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.855217 13973 raft_consensus.cc:504] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.856392 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:23.856611 12962 raft_consensus.cc:3037] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.857157 14064 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240502 14:17:23.859419 12962 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:23.860524 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:23.860764 12703 raft_consensus.cc:3037] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.861008 14066 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:23.861223 14066 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.027s user 0.005s sys 0.003s
I20240502 14:17:23.862042 14066 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.862367 14066 raft_consensus.cc:374] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.862510 14066 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.862841 14066 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.863725 12703 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:23.864956 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:23.865257 12833 raft_consensus.cc:3037] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.866282 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:23.866490 12309 raft_consensus.cc:3037] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.868906 12833 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:23.869192 12309 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:23.869505 12377 leader_election.cc:302] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:23.870388 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:23.870617 12571 raft_consensus.cc:3037] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.874341 12571 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:23.875536 13973 leader_election.cc:288] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 election: Requested vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:23.875746 13973 raft_consensus.cc:2781] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:23.875876 13973 raft_consensus.cc:686] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Becoming Leader. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Running, Role: LEADER
I20240502 14:17:23.876157 14066 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.015s user 0.002s sys 0.000s
I20240502 14:17:23.876194 13973 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.865469 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c9ff45109a6d4dc9aba4617819929981" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:23.880461 13095 raft_consensus.cc:3037] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.880951 14064 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:23.883134 13095 raft_consensus.cc:2454] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:23.885169 14064 tablet_bootstrap.cc:654] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.889273 14077 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:23.889513 14077 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.037s user 0.011s sys 0.023s
I20240502 14:17:23.889313 12165 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: term changed from 0 to 1, leader changed from <none> to eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130). New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:23.890364 14077 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.890638 14077 raft_consensus.cc:374] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.890726 14077 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.890990 14077 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.892014 14077 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:23.892359 14077 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:23.892885 14064 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:23.893086 14064 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.001s
I20240502 14:17:23.893808 14064 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.894048 14064 raft_consensus.cc:374] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.894129 14064 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.894480 14077 tablet_bootstrap.cc:654] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.894407 14064 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.895156 14064 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:23.895433 14064 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:23.896899 14064 tablet_bootstrap.cc:654] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.898506 14077 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:23.898658 14077 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.005s
I20240502 14:17:23.899394 14077 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.899660 14077 raft_consensus.cc:374] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.899767 14077 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.899979 14077 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.900894 14077 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.002s user 0.001s sys 0.002s
I20240502 14:17:23.902122 14078 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:23.902344 14078 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.074s user 0.000s sys 0.009s
I20240502 14:17:23.902905 14064 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:23.903051 14064 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240502 14:17:23.903162 14078 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.903491 14078 raft_consensus.cc:374] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.903585 14078 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.903843 14078 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.903909 14064 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.904160 14064 raft_consensus.cc:374] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.904254 14064 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.904529 14078 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:23.904529 14064 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.904871 14078 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:23.905750 14064 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 14:17:23.906438 14078 tablet_bootstrap.cc:654] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:23.909485 14078 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:23.909658 14078 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 14:17:23.910238 14078 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.910424 14078 raft_consensus.cc:374] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:23.910518 14078 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:23.910748 14078 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.911270 14078 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:23.960399 14044 raft_consensus.cc:482] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:23.960645 14044 raft_consensus.cc:504] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.961805 14044 leader_election.cc:288] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:23.961927 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:23.961967 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:23.962080 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:23.962170 12962 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:23.962240 12703 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:23.962296 12833 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:23.962657 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:23.962917 12309 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:23.962889 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:23.963135 12571 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:23.963339 13031 leader_election.cc:302] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:23.963455 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:23.963610 14044 raft_consensus.cc:2781] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:23.963706 14044 raft_consensus.cc:482] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:23.963737 12440 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:23.963789 14044 raft_consensus.cc:3037] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.965910 14044 raft_consensus.cc:504] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.967032 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:23.967135 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:23.967154 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:23.967206 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:23.967272 12962 raft_consensus.cc:3037] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.967346 12703 raft_consensus.cc:3037] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.967367 12833 raft_consensus.cc:3037] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.967401 12309 raft_consensus.cc:3037] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.970090 12962 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:23.970196 12309 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:23.970196 12703 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:23.970220 12833 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:23.970649 13031 leader_election.cc:302] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:23.971302 14044 leader_election.cc:288] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Requested vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:23.971580 14096 raft_consensus.cc:2781] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:23.971848 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:23.972101 12440 raft_consensus.cc:3037] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.972368 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "01ffac38da1d4ece9cba649a59d5518b" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:23.972606 12571 raft_consensus.cc:3037] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:23.974321 12440 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:23.974726 14096 raft_consensus.cc:686] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Becoming Leader. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Running, Role: LEADER
I20240502 14:17:23.975184 14096 consensus_queue.cc:237] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:23.975775 12571 raft_consensus.cc:2454] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:23.981918 12167 catalog_manager.cc:5484] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: term changed from 0 to 1, leader changed from <none> to 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135). New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:24.045704 14096 raft_consensus.cc:482] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:24.045905 14096 raft_consensus.cc:504] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:24.046137 14045 raft_consensus.cc:482] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:24.046330 14045 raft_consensus.cc:504] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:24.047191 14096 leader_election.cc:288] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:24.047242 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:24.047489 12703 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:24.047638 14045 leader_election.cc:288] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:24.048327 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:24.048319 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:24.048571 12440 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:24.048622 12307 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:24.048591 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:24.048848 12832 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:24.048866 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:24.048179 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:24.049065 13095 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:24.049147 12309 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:24.048540 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:24.049662 12703 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:24.049639 12507 leader_election.cc:302] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:24.049672 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:24.049947 12962 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:24.050081 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:24.050118 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:24.050302 12964 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:24.050415 12833 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:24.050469 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:24.050668 12571 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:24.050941 13032 leader_election.cc:302] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters:
I20240502 14:17:24.051237 14097 raft_consensus.cc:2781] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:24.051442 14045 raft_consensus.cc:2781] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:24.051419 14097 raft_consensus.cc:482] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:24.051456 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:24.051559 14045 raft_consensus.cc:482] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:24.051657 14045 raft_consensus.cc:3037] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:24.051647 12440 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:24.051777 14097 raft_consensus.cc:3037] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:24.054662 14045 raft_consensus.cc:504] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:24.054657 14097 raft_consensus.cc:504] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:24.055922 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:24.055861 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:24.056020 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:24.056106 12703 raft_consensus.cc:3037] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:24.056020 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:24.056262 12832 raft_consensus.cc:3037] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:24.057149 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:24.057344 12964 raft_consensus.cc:3037] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:24.057538 14045 leader_election.cc:288] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Requested vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:24.058059 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:24.058272 13095 raft_consensus.cc:2379] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:24.058851 12703 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:24.058997 12832 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:24.059634 12307 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:24.059742 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:24.059883 12307 raft_consensus.cc:3037] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:24.059971 14097 leader_election.cc:288] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Requested vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:24.060071 12964 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:24.060721 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:24.060951 12571 raft_consensus.cc:2379] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:24.060926 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:24.061126 12440 raft_consensus.cc:3037] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:24.061561 13029 leader_election.cc:302] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, d693e089cfa54fbb8df91939410b19db; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f
I20240502 14:17:24.062012 14097 raft_consensus.cc:2726] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 14:17:24.061272 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:24.062536 12964 raft_consensus.cc:2379] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:24.062605 12307 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:24.063316 12507 leader_election.cc:302] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, d693e089cfa54fbb8df91939410b19db
I20240502 14:17:24.063419 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:24.063633 14045 raft_consensus.cc:2781] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:24.063899 14045 raft_consensus.cc:686] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Becoming Leader. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Running, Role: LEADER
I20240502 14:17:24.064128 12440 raft_consensus.cc:2454] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:24.064250 14045 consensus_queue.cc:237] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:24.067720 12167 catalog_manager.cc:5484] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: term changed from 0 to 1, leader changed from <none> to 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:24.275825 14090 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.302863 14090 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.322528 14090 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.332780 14090 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.338054 14106 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.366912 14097 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.371753 14106 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.433276 14097 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:24.467900 14100 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:24.468231 14100 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:24.476119 14100 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:17:24.496515 14097 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.503019 14097 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.506559 14097 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_3'
I20240502 14:17:24.518128 14097 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.529305 14045 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.542493 14045 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.543118 14045 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.548691 14045 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.549196 14045 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:24.560289 14059 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 530.229 ms
time per row: 0.265115 ms
I20240502 14:17:25.215852 12165 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60530:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:25.216631 12165 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_4 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 14:17:25.226019 12945 tablet_service.cc:1467] Processing CreateTablet for tablet 5614c0ba0e374af1853bd2fa586d3a76 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.226358 12813 tablet_service.cc:1467] Processing CreateTablet for tablet 5614c0ba0e374af1853bd2fa586d3a76 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.226522 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5614c0ba0e374af1853bd2fa586d3a76. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.226544 12681 tablet_service.cc:1467] Processing CreateTablet for tablet 5614c0ba0e374af1853bd2fa586d3a76 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.226797 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5614c0ba0e374af1853bd2fa586d3a76. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.226927 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5614c0ba0e374af1853bd2fa586d3a76. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.232146 14220 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:25.233017 12420 tablet_service.cc:1467] Processing CreateTablet for tablet 5614c0ba0e374af1853bd2fa586d3a76 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.233799 12551 tablet_service.cc:1467] Processing CreateTablet for tablet f358e8fda9ca4eeeb1dee5b2011ac029 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.233970 14220 tablet_bootstrap.cc:654] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.233909 12421 tablet_service.cc:1467] Processing CreateTablet for tablet b830809623f543438208dc87476d18c6 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.234234 12551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.234366 12550 tablet_service.cc:1467] Processing CreateTablet for tablet 5614c0ba0e374af1853bd2fa586d3a76 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.234612 12419 tablet_service.cc:1467] Processing CreateTablet for tablet f358e8fda9ca4eeeb1dee5b2011ac029 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.234736 12550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5614c0ba0e374af1853bd2fa586d3a76. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.237057 12944 tablet_service.cc:1467] Processing CreateTablet for tablet b830809623f543438208dc87476d18c6 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.233479 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5614c0ba0e374af1853bd2fa586d3a76. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.237421 12944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.238540 12288 tablet_service.cc:1467] Processing CreateTablet for tablet b830809623f543438208dc87476d18c6 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.238768 12419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.238972 12288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.238994 13076 tablet_service.cc:1467] Processing CreateTablet for tablet 5614c0ba0e374af1853bd2fa586d3a76 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.239401 13076 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5614c0ba0e374af1853bd2fa586d3a76. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.241883 12421 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.241847 13075 tablet_service.cc:1467] Processing CreateTablet for tablet f358e8fda9ca4eeeb1dee5b2011ac029 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.242185 13075 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.242165 12943 tablet_service.cc:1467] Processing CreateTablet for tablet f358e8fda9ca4eeeb1dee5b2011ac029 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.242516 12943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.246028 12289 tablet_service.cc:1467] Processing CreateTablet for tablet 5614c0ba0e374af1853bd2fa586d3a76 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.246464 12289 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5614c0ba0e374af1853bd2fa586d3a76. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.246963 12812 tablet_service.cc:1467] Processing CreateTablet for tablet b830809623f543438208dc87476d18c6 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.247478 12812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.249898 12287 tablet_service.cc:1467] Processing CreateTablet for tablet f358e8fda9ca4eeeb1dee5b2011ac029 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.249979 12814 tablet_service.cc:1467] Processing CreateTablet for tablet f358e8fda9ca4eeeb1dee5b2011ac029 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.250284 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.250433 12814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.254833 12683 tablet_service.cc:1467] Processing CreateTablet for tablet f358e8fda9ca4eeeb1dee5b2011ac029 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.255290 12683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.238520 12552 tablet_service.cc:1467] Processing CreateTablet for tablet b830809623f543438208dc87476d18c6 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.256563 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.257644 12682 tablet_service.cc:1467] Processing CreateTablet for tablet b830809623f543438208dc87476d18c6 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.257997 12682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.267879 13074 tablet_service.cc:1467] Processing CreateTablet for tablet b830809623f543438208dc87476d18c6 (DEFAULT_TABLE table=rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:25.268335 13074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:25.279397 14225 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:25.281303 14225 tablet_bootstrap.cc:654] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.285831 14221 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:25.291301 14227 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:25.293603 14227 tablet_bootstrap.cc:654] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.296607 14220 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:25.296802 14220 ts_tablet_manager.cc:1367] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.065s user 0.014s sys 0.014s
I20240502 14:17:25.297922 14226 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:25.299444 14220 raft_consensus.cc:348] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.299836 14226 tablet_bootstrap.cc:654] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.299808 14220 raft_consensus.cc:374] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.300598 14220 raft_consensus.cc:729] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.301163 14220 consensus_queue.cc:260] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.301908 14224 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:25.303333 14220 ts_tablet_manager.cc:1398] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.006s user 0.002s sys 0.002s
I20240502 14:17:25.303624 14220 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:25.305375 14222 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:25.305809 14225 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:25.306006 14225 ts_tablet_manager.cc:1367] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.027s user 0.001s sys 0.003s
I20240502 14:17:25.306404 14220 tablet_bootstrap.cc:654] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.306988 14225 raft_consensus.cc:348] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.307358 14225 raft_consensus.cc:374] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.307453 14225 raft_consensus.cc:729] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.308576 14221 tablet_bootstrap.cc:654] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.311653 14222 tablet_bootstrap.cc:654] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.313459 14224 tablet_bootstrap.cc:654] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.314420 14225 consensus_queue.cc:260] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.315011 14227 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:25.315214 14227 ts_tablet_manager.cc:1367] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.024s user 0.014s sys 0.007s
I20240502 14:17:25.315315 14225 ts_tablet_manager.cc:1398] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240502 14:17:25.316176 14225 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:25.316390 14227 raft_consensus.cc:348] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.316804 14227 raft_consensus.cc:374] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.316953 14227 raft_consensus.cc:729] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.317802 14225 tablet_bootstrap.cc:654] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.318182 14227 consensus_queue.cc:260] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.318990 14227 ts_tablet_manager.cc:1398] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240502 14:17:25.324985 14224 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:25.325192 14224 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.023s user 0.002s sys 0.001s
I20240502 14:17:25.326514 14227 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:25.329142 14227 tablet_bootstrap.cc:654] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.330590 14222 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:25.331169 14222 ts_tablet_manager.cc:1367] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.028s user 0.005s sys 0.000s
I20240502 14:17:25.332070 14226 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:25.332325 14226 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.035s user 0.006s sys 0.000s
I20240502 14:17:25.333406 14226 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.334050 14226 raft_consensus.cc:374] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.334214 14226 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.334887 14221 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:25.334620 14226 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.335161 14221 ts_tablet_manager.cc:1367] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.049s user 0.010s sys 0.009s
I20240502 14:17:25.335767 14226 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:25.336117 14226 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:25.336637 14221 raft_consensus.cc:348] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.337157 14221 raft_consensus.cc:374] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.337369 14221 raft_consensus.cc:729] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.337852 14221 consensus_queue.cc:260] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.344071 14221 ts_tablet_manager.cc:1398] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.009s user 0.004s sys 0.000s
I20240502 14:17:25.345654 14221 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:25.348140 14221 tablet_bootstrap.cc:654] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.353821 14224 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.355428 14224 raft_consensus.cc:374] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.355557 14224 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.356477 14227 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:25.356768 14227 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.030s user 0.007s sys 0.019s
I20240502 14:17:25.357789 14227 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.358278 14227 raft_consensus.cc:374] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.358410 14227 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.358736 14227 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.360875 14227 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:17:25.361265 14227 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:25.363721 14221 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:25.363991 14221 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.018s user 0.005s sys 0.000s
I20240502 14:17:25.364216 14227 tablet_bootstrap.cc:654] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.365175 14221 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.365732 14221 raft_consensus.cc:374] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.365892 14221 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.366211 14224 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.366437 14221 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.358219 14222 raft_consensus.cc:348] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.367101 14222 raft_consensus.cc:374] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.367272 14222 raft_consensus.cc:729] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.368481 14221 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:17:25.368813 14221 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:25.371783 14221 tablet_bootstrap.cc:654] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.377172 14227 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:25.377351 14227 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.004s
I20240502 14:17:25.378310 14227 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.378731 14227 raft_consensus.cc:374] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.378861 14227 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.379175 14227 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.380513 14227 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240502 14:17:25.384099 14220 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:25.384333 14220 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.081s user 0.004s sys 0.033s
I20240502 14:17:25.385107 14220 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.385409 14220 raft_consensus.cc:374] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.385473 14220 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.385769 14220 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.386358 14222 consensus_queue.cc:260] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.386828 14226 tablet_bootstrap.cc:654] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.387010 14221 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:25.387051 14220 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:25.387184 14221 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.003s
I20240502 14:17:25.387455 14220 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:25.388099 14222 ts_tablet_manager.cc:1398] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.048s user 0.002s sys 0.000s
I20240502 14:17:25.388163 14221 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.388921 14221 raft_consensus.cc:374] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.389031 14221 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.389328 14222 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:25.389317 14221 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.390116 14221 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:25.391281 14225 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:25.391489 14225 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.075s user 0.009s sys 0.021s
I20240502 14:17:25.392146 14222 tablet_bootstrap.cc:654] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.393406 14220 tablet_bootstrap.cc:654] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.394551 14225 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.394845 14225 raft_consensus.cc:374] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.394925 14225 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.395179 14225 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.396513 14225 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.005s user 0.001s sys 0.002s
I20240502 14:17:25.396829 14225 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:25.396818 14224 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.071s user 0.002s sys 0.002s
I20240502 14:17:25.397168 14224 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:25.399360 14225 tablet_bootstrap.cc:654] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.399463 14224 tablet_bootstrap.cc:654] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.401818 14222 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:25.403112 14222 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.001s
I20240502 14:17:25.403879 14222 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.404122 14222 raft_consensus.cc:374] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.404212 14222 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.404954 14222 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.405402 14224 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:25.405660 14224 ts_tablet_manager.cc:1367] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.002s
I20240502 14:17:25.405833 14222 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:17:25.406162 14222 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:25.406912 14224 raft_consensus.cc:348] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.407330 14224 raft_consensus.cc:374] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.407465 14224 raft_consensus.cc:729] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.407881 14222 tablet_bootstrap.cc:654] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.407822 14224 consensus_queue.cc:260] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.408664 14224 ts_tablet_manager.cc:1398] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:25.409065 14226 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:25.409304 14224 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:25.409305 14226 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.073s user 0.005s sys 0.000s
I20240502 14:17:25.409926 14220 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:25.410122 14220 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.023s user 0.007s sys 0.000s
I20240502 14:17:25.410238 14226 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.410632 14226 raft_consensus.cc:374] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.410727 14226 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.410979 14220 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.411264 14220 raft_consensus.cc:374] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.411381 14220 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.411710 14220 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.412283 14224 tablet_bootstrap.cc:654] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.412083 14226 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.413035 14226 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.004s user 0.000s sys 0.003s
I20240502 14:17:25.413470 14226 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:25.414026 14220 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:17:25.419974 14195 raft_consensus.cc:482] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:25.420143 14195 raft_consensus.cc:504] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.422010 14195 leader_election.cc:288] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:25.422843 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:25.423076 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:25.423393 12833 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.424293 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:25.424574 12701 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.424899 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:25.425166 13095 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.425733 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:25.426071 12309 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
W20240502 14:17:25.426618 12898 leader_election.cc:341] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:25.428030 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:25.428364 12441 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.440745 14226 tablet_bootstrap.cc:654] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:25.445341 14225 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:25.445626 14225 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.049s user 0.011s sys 0.010s
I20240502 14:17:25.446630 14225 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.450708 14225 raft_consensus.cc:374] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.450841 14225 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.451179 14225 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.453261 14225 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 14:17:25.458328 14224 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:25.458540 14224 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.050s user 0.029s sys 0.009s
I20240502 14:17:25.459517 14224 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.459815 14224 raft_consensus.cc:374] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.459934 14224 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.459753 12900 leader_election.cc:302] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874
I20240502 14:17:25.460227 14224 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.460412 14249 raft_consensus.cc:2781] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:25.460523 14249 raft_consensus.cc:482] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:25.460638 14249 raft_consensus.cc:3037] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.461851 14224 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.003s user 0.002s sys 0.001s
I20240502 14:17:25.462477 14222 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:25.462638 14222 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.057s user 0.000s sys 0.005s
I20240502 14:17:25.468786 14045 raft_consensus.cc:482] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:25.468978 14045 raft_consensus.cc:504] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.464413 14195 raft_consensus.cc:482] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:25.469774 14195 raft_consensus.cc:504] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.470257 14045 leader_election.cc:288] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:25.471062 14195 leader_election.cc:288] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:25.471602 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:25.471678 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:25.471877 12308 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.472008 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:25.472256 12964 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:25.472208 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:25.472482 12703 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.472788 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:25.472817 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:25.473050 12833 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.473297 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:25.473348 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:25.473541 12701 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:25.473909 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:25.474079 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:25.474143 13095 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.474279 12572 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:25.474460 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:25.474659 13096 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:25.474814 12832 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
W20240502 14:17:25.475536 12506 leader_election.cc:341] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:25.463512 14222 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.476198 14222 raft_consensus.cc:374] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.476171 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:25.476323 14222 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.468101 14249 raft_consensus.cc:504] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
W20240502 14:17:25.476769 12899 leader_election.cc:341] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:25.476645 14222 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.477636 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:25.477805 14222 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.015s user 0.000s sys 0.002s
I20240502 14:17:25.477838 12832 raft_consensus.cc:3037] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.478379 12507 leader_election.cc:302] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters: 70a425a77fe54cbcb9755f9c540fd367, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:25.478693 14168 raft_consensus.cc:2781] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:25.478801 14168 raft_consensus.cc:482] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:25.478905 14168 raft_consensus.cc:3037] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.479233 14226 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:25.479405 14226 ts_tablet_manager.cc:1367] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.066s user 0.003s sys 0.002s
I20240502 14:17:25.480232 14226 raft_consensus.cc:348] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.480545 14226 raft_consensus.cc:374] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:25.480649 14226 raft_consensus.cc:729] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:25.480790 12832 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:25.480957 14226 consensus_queue.cc:260] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.481804 14226 ts_tablet_manager.cc:1398] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240502 14:17:25.482682 14249 leader_election.cc:288] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Requested vote from peers c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:25.483592 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:25.483836 12572 raft_consensus.cc:3037] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.485445 12900 leader_election.cc:302] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters: eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:25.486636 12572 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:25.487644 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:25.487651 14168 raft_consensus.cc:504] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.487912 12308 raft_consensus.cc:3037] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.488209 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:25.488433 13096 raft_consensus.cc:3037] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.489351 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:25.489596 12834 raft_consensus.cc:3037] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.491158 13096 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:25.491650 12308 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:25.491739 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:25.491963 12703 raft_consensus.cc:3037] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.492424 12834 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:25.492547 12900 leader_election.cc:302] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:25.492990 14168 leader_election.cc:288] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Requested vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:25.493425 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:25.493669 12308 raft_consensus.cc:3037] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.493696 14195 raft_consensus.cc:2781] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:25.493791 14195 raft_consensus.cc:482] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:25.493899 14195 raft_consensus.cc:3037] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.494737 12703 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:25.495888 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:25.496055 12701 raft_consensus.cc:3037] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.496628 14195 raft_consensus.cc:504] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.497077 12308 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:25.497454 14249 raft_consensus.cc:2781] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:25.498023 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:25.498317 12703 raft_consensus.cc:2379] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:25.498741 12701 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:25.499410 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:25.499662 12834 raft_consensus.cc:2379] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:25.500203 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:25.500468 12440 raft_consensus.cc:3037] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.500850 14195 leader_election.cc:288] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Requested vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:25.502486 12440 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:25.503477 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:25.503787 13096 raft_consensus.cc:3037] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.505503 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b830809623f543438208dc87476d18c6" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:25.505730 12441 raft_consensus.cc:3037] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.506707 13096 raft_consensus.cc:2454] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:25.507879 12441 raft_consensus.cc:2454] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:25.510377 12507 leader_election.cc:302] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:25.510684 14168 raft_consensus.cc:2781] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:25.510877 14168 raft_consensus.cc:686] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Becoming Leader. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Running, Role: LEADER
I20240502 14:17:25.511232 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:25.511200 14168 consensus_queue.cc:237] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.511468 12440 raft_consensus.cc:2379] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:25.512147 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:25.513190 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:25.513259 12898 leader_election.cc:302] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: d693e089cfa54fbb8df91939410b19db; no voters: 53479f0fb09c454e9b3cbe7398bf8874, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:25.513489 12308 raft_consensus.cc:2379] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:25.514092 14195 raft_consensus.cc:2726] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 14:17:25.514524 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:25.514832 12964 raft_consensus.cc:2379] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate d693e089cfa54fbb8df91939410b19db in this term.
I20240502 14:17:25.517324 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "5614c0ba0e374af1853bd2fa586d3a76" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:25.517995 13096 raft_consensus.cc:2379] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:25.521389 14249 raft_consensus.cc:686] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Becoming Leader. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Running, Role: LEADER
I20240502 14:17:25.521793 14249 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.523885 12167 catalog_manager.cc:5484] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: term changed from 0 to 1, leader changed from <none> to 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:25.529062 12165 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db reported cstate change: term changed from 0 to 1, leader changed from <none> to d693e089cfa54fbb8df91939410b19db (127.27.220.134). New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:25.546237 14173 raft_consensus.cc:482] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:25.546448 14173 raft_consensus.cc:504] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.547614 14173 leader_election.cc:288] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:25.547735 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:25.547745 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:25.547829 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:25.547994 12834 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:25.547994 12703 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:25.548084 12964 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:25.548594 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:25.548602 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:25.548831 12572 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:25.548887 13096 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:25.548991 12377 leader_election.cc:302] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:25.549014 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:25.549261 12308 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 0.
I20240502 14:17:25.549402 14173 raft_consensus.cc:2781] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:25.549417 14168 raft_consensus.cc:482] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:25.549567 14173 raft_consensus.cc:482] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:25.549681 14173 raft_consensus.cc:3037] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.549600 14168 raft_consensus.cc:504] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.550745 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:25.550818 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:25.550843 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:25.551044 12964 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:25.551044 12834 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:25.551085 12703 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:25.551582 12507 leader_election.cc:302] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db; no voters:
I20240502 14:17:25.551903 14045 raft_consensus.cc:2781] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:25.552047 14045 raft_consensus.cc:482] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:25.552145 14045 raft_consensus.cc:3037] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.552268 14168 leader_election.cc:288] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:25.552290 14173 raft_consensus.cc:504] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.552753 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:25.552956 12308 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:25.553040 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:25.553267 12440 raft_consensus.cc:2379] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:25.553594 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:25.553810 12703 raft_consensus.cc:3037] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.553995 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:25.554296 13096 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:25.554807 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:25.555020 12964 raft_consensus.cc:3037] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.554983 14045 raft_consensus.cc:504] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.555521 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:25.555716 12834 raft_consensus.cc:3037] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.556228 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:25.556612 12703 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:25.557181 14045 leader_election.cc:288] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Requested vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:25.557404 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:25.557595 14173 leader_election.cc:288] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 election: Requested vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:25.558068 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:25.558313 12572 raft_consensus.cc:2379] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate eddeff583cf34c8dab2942fb36abbf9e in current term 1: Already voted for candidate 53479f0fb09c454e9b3cbe7398bf8874 in this term.
I20240502 14:17:25.558665 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:25.558902 12440 raft_consensus.cc:2379] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:25.558960 12964 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:25.558951 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:25.559131 12308 raft_consensus.cc:3037] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.559131 13095 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:25.559317 13095 raft_consensus.cc:3037] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:25.559530 12834 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:25.560197 12377 leader_election.cc:302] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874
I20240502 14:17:25.561295 14173 raft_consensus.cc:2781] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:25.561597 14173 raft_consensus.cc:686] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Becoming Leader. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Running, Role: LEADER
I20240502 14:17:25.561800 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:25.562029 12308 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eddeff583cf34c8dab2942fb36abbf9e in term 1.
I20240502 14:17:25.561993 14173 consensus_queue.cc:237] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:25.562641 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:25.562938 12703 raft_consensus.cc:2379] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate eddeff583cf34c8dab2942fb36abbf9e in this term.
I20240502 14:17:25.563037 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f358e8fda9ca4eeeb1dee5b2011ac029" candidate_uuid: "eddeff583cf34c8dab2942fb36abbf9e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:25.563067 12507 leader_election.cc:302] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874; no voters: 70a425a77fe54cbcb9755f9c540fd367, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:25.564283 13095 raft_consensus.cc:2454] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:25.564604 14045 raft_consensus.cc:2726] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 14:17:25.568403 12165 catalog_manager.cc:5484] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: term changed from 0 to 1, leader changed from <none> to eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130). New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:25.907737 14249 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:25.936959 14168 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:25.956187 14173 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:25.959846 14168 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:25.976984 14171 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:25.999791 14168 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.014837 14268 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.020287 14267 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.020792 14267 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.022408 14173 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.039731 14249 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.048303 14249 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:26.052969 14256 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:26.053704 14256 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:26.069317 14256 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:17:26.079027 14204 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.079469 14204 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.087234 14249 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.111547 14204 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.147420 14252 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:26.149787 14252 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_4'
W20240502 14:17:26.533416 14308 outbound_call.cc:322] RPC callback for RPC call kudu.tserver.TabletServerService.Write -> {remote=127.27.220.134:36329, user_credentials={real_user=slave}} blocked reactor thread for 43503.2us
INSERT report
rows total: 2000
time total: 439.966 ms
time per row: 0.219983 ms
I20240502 14:17:26.772717 12165 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60530:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:26.773448 12165 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_5 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 14:17:26.783802 12419 tablet_service.cc:1467] Processing CreateTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.784376 12419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.784535 12550 tablet_service.cc:1467] Processing CreateTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.784982 12550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.785882 12288 tablet_service.cc:1467] Processing CreateTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.786223 12288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.789980 12945 tablet_service.cc:1467] Processing CreateTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.790423 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.791334 14368 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:26.791924 12287 tablet_service.cc:1467] Processing CreateTablet for tablet 9cd04934b5264d7a9643974c0386cac4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.792354 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.793291 14368 tablet_bootstrap.cc:654] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.794010 12943 tablet_service.cc:1467] Processing CreateTablet for tablet 9cd04934b5264d7a9643974c0386cac4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.794512 12943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.798358 12681 tablet_service.cc:1467] Processing CreateTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.798609 12420 tablet_service.cc:1467] Processing CreateTablet for tablet 9cd04934b5264d7a9643974c0386cac4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.798573 12683 tablet_service.cc:1467] Processing CreateTablet for tablet 9cd04934b5264d7a9643974c0386cac4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.798836 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.798996 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.799083 12683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.802908 13076 tablet_service.cc:1467] Processing CreateTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.803340 13076 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.803823 12812 tablet_service.cc:1467] Processing CreateTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.804288 12812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.805919 13074 tablet_service.cc:1467] Processing CreateTablet for tablet 9cd04934b5264d7a9643974c0386cac4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.806293 13074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.808827 12682 tablet_service.cc:1467] Processing CreateTablet for tablet f8526552032e4a45ab4be4743f12abd0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.809218 12682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8526552032e4a45ab4be4743f12abd0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.809386 12814 tablet_service.cc:1467] Processing CreateTablet for tablet 9cd04934b5264d7a9643974c0386cac4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.809891 12814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.809921 12552 tablet_service.cc:1467] Processing CreateTablet for tablet 9cd04934b5264d7a9643974c0386cac4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.810355 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.818403 12944 tablet_service.cc:1467] Processing CreateTablet for tablet f8526552032e4a45ab4be4743f12abd0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.818858 12944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8526552032e4a45ab4be4743f12abd0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.823537 12813 tablet_service.cc:1467] Processing CreateTablet for tablet f8526552032e4a45ab4be4743f12abd0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.823983 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8526552032e4a45ab4be4743f12abd0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.824277 13075 tablet_service.cc:1467] Processing CreateTablet for tablet f8526552032e4a45ab4be4743f12abd0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.824738 13075 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8526552032e4a45ab4be4743f12abd0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.824954 12419 tablet_service.cc:1467] Processing CreateTablet for tablet f8526552032e4a45ab4be4743f12abd0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.826592 12419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8526552032e4a45ab4be4743f12abd0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.828096 12288 tablet_service.cc:1467] Processing CreateTablet for tablet f8526552032e4a45ab4be4743f12abd0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.828574 12288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8526552032e4a45ab4be4743f12abd0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.838418 14373 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:26.842381 14371 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:26.842715 14375 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:26.844246 14371 tablet_bootstrap.cc:654] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.844313 14375 tablet_bootstrap.cc:654] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.845999 14374 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:26.845933 12552 tablet_service.cc:1467] Processing CreateTablet for tablet f8526552032e4a45ab4be4743f12abd0 (DEFAULT_TABLE table=rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:26.846357 14373 tablet_bootstrap.cc:654] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.846398 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8526552032e4a45ab4be4743f12abd0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:26.849295 14374 tablet_bootstrap.cc:654] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.858497 14372 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:26.860914 14374 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:26.861140 14374 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.004s
I20240502 14:17:26.861452 14371 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:26.861727 14372 tablet_bootstrap.cc:654] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.861917 14371 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.020s user 0.004s sys 0.000s
I20240502 14:17:26.862092 14374 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.862388 14374 raft_consensus.cc:374] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.862471 14374 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.862735 14374 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.863395 14374 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:26.863929 14371 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.864825 14371 raft_consensus.cc:374] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.865090 14371 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.865559 14371 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.866416 14375 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:26.866637 14375 ts_tablet_manager.cc:1367] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.024s user 0.002s sys 0.005s
I20240502 14:17:26.866758 14369 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:26.867628 14371 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240502 14:17:26.867513 14375 raft_consensus.cc:348] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.867856 14375 raft_consensus.cc:374] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.868381 14375 raft_consensus.cc:729] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.868708 14373 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:26.868865 14373 ts_tablet_manager.cc:1367] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.032s user 0.007s sys 0.000s
I20240502 14:17:26.868724 14375 consensus_queue.cc:260] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.869449 14369 tablet_bootstrap.cc:654] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.869808 14371 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:26.870280 14373 raft_consensus.cc:348] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.870949 14373 raft_consensus.cc:374] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.871253 14373 raft_consensus.cc:729] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.871336 14371 tablet_bootstrap.cc:654] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.872542 14373 consensus_queue.cc:260] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.873711 14375 ts_tablet_manager.cc:1398] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240502 14:17:26.874106 14375 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:26.879770 14375 tablet_bootstrap.cc:654] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.883345 14374 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:26.885442 14374 tablet_bootstrap.cc:654] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.891387 14371 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:26.891554 14368 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:26.891582 14371 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.022s user 0.002s sys 0.010s
I20240502 14:17:26.891716 14368 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.100s user 0.008s sys 0.013s
I20240502 14:17:26.892223 14371 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.892462 14371 raft_consensus.cc:374] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.892570 14371 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.892475 14368 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.893045 14368 raft_consensus.cc:374] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.892874 14371 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.893148 14368 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.893395 14368 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.894178 14371 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:17:26.894291 14375 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:26.894521 14375 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.020s user 0.004s sys 0.001s
I20240502 14:17:26.894608 14371 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:26.895154 14373 ts_tablet_manager.cc:1398] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.026s user 0.004s sys 0.000s
I20240502 14:17:26.895892 14374 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:26.896387 14371 tablet_bootstrap.cc:654] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.896910 14369 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:26.897094 14369 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.030s user 0.002s sys 0.010s
I20240502 14:17:26.897186 14372 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:26.897435 14372 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.040s user 0.001s sys 0.004s
I20240502 14:17:26.898113 14369 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.898320 14372 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.898592 14369 raft_consensus.cc:374] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.898697 14372 raft_consensus.cc:374] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.898809 14369 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.898871 14372 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.899235 14372 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.899224 14369 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.899930 14372 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:26.900187 14369 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.003s user 0.001s sys 0.002s
I20240502 14:17:26.901945 14372 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:26.903404 14373 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:26.904834 14371 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:26.905087 14371 ts_tablet_manager.cc:1367] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.011s user 0.008s sys 0.002s
I20240502 14:17:26.905989 14373 tablet_bootstrap.cc:654] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.906137 14371 raft_consensus.cc:348] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.906077 14375 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.906440 14375 raft_consensus.cc:374] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.906510 14371 raft_consensus.cc:374] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.906635 14371 raft_consensus.cc:729] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.906858 14371 consensus_queue.cc:260] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.907615 14371 ts_tablet_manager.cc:1398] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:26.907662 14369 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:26.907995 14374 ts_tablet_manager.cc:1367] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.028s user 0.004s sys 0.001s
I20240502 14:17:26.908165 14375 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.908540 14375 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.908864 14374 raft_consensus.cc:348] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.909225 14374 raft_consensus.cc:374] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.909344 14374 raft_consensus.cc:729] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.909672 14374 consensus_queue.cc:260] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.910265 14372 tablet_bootstrap.cc:654] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.911252 14374 ts_tablet_manager.cc:1398] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 14:17:26.911572 14374 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:26.914494 14374 tablet_bootstrap.cc:654] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.917202 14375 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.022s user 0.002s sys 0.000s
I20240502 14:17:26.922791 14375 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:26.924589 14369 tablet_bootstrap.cc:654] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.933172 14375 tablet_bootstrap.cc:654] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.938719 14386 raft_consensus.cc:482] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:26.938948 14368 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.047s user 0.005s sys 0.010s
I20240502 14:17:26.939298 14368 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:26.938944 14386 raft_consensus.cc:504] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.941309 14368 tablet_bootstrap.cc:654] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.942032 14252 raft_consensus.cc:482] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:26.942215 14252 raft_consensus.cc:504] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.943508 14252 leader_election.cc:288] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:26.944787 12702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:26.944823 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:26.945082 13096 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
W20240502 14:17:26.945801 12900 leader_election.cc:341] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:26.946516 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:26.946801 12440 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:26.947594 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
W20240502 14:17:26.948220 12898 leader_election.cc:341] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:26.949469 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:26.949774 12834 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:26.950414 12901 leader_election.cc:302] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874, af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:26.951048 14324 raft_consensus.cc:2781] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:26.951151 14324 raft_consensus.cc:482] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:26.951261 14324 raft_consensus.cc:3037] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:26.955507 14375 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:26.955940 14368 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:26.955852 14324 raft_consensus.cc:504] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.956151 14368 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.003s
I20240502 14:17:26.956936 14375 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.034s user 0.005s sys 0.002s
I20240502 14:17:26.957052 14368 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.957378 14368 raft_consensus.cc:374] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.957468 14368 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.957301 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:26.957876 12834 raft_consensus.cc:3037] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:26.958271 12702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:26.958778 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:26.958990 13096 raft_consensus.cc:3037] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:26.958032 14368 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.959455 14368 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:26.960767 14368 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:26.961398 12834 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
W20240502 14:17:26.961923 12900 leader_election.cc:341] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:26.963094 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:26.963342 12440 raft_consensus.cc:3037] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:26.963390 14368 tablet_bootstrap.cc:654] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.963295 14375 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.963698 14375 raft_consensus.cc:374] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.966151 14324 leader_election.cc:288] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Requested vote from peers c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:26.966506 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
W20240502 14:17:26.967061 12898 leader_election.cc:341] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:26.967288 14374 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:26.967440 14374 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.056s user 0.015s sys 0.035s
I20240502 14:17:26.968107 12440 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:26.968103 14374 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.968331 14374 raft_consensus.cc:374] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.968453 14374 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.968727 14374 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.969269 14374 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:26.969359 13096 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:26.969918 12900 leader_election.cc:302] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874, af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:26.970299 14324 raft_consensus.cc:2781] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:26.963812 14375 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.970924 14324 raft_consensus.cc:686] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Becoming Leader. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Running, Role: LEADER
I20240502 14:17:26.971017 14375 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.971446 14324 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.973121 14375 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.011s user 0.001s sys 0.001s
I20240502 14:17:26.977182 14372 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:26.977408 14372 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.076s user 0.000s sys 0.004s
I20240502 14:17:26.978370 14372 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.978775 14372 raft_consensus.cc:374] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:26.978865 14372 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:26.979125 14372 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.981878 14372 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:17:26.982167 12306 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:26.982383 14372 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:26.982396 12306 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:26.982861 12306 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:26.983065 12306 raft_consensus.cc:3037] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:26.984652 14372 tablet_bootstrap.cc:654] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:26.986686 12306 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:26.988255 14263 raft_consensus.cc:482] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:26.988265 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:26.988515 12441 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:26.988426 14263 raft_consensus.cc:504] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.989760 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:26.989789 12702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:26.989910 14263 leader_election.cc:288] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:26.990000 12571 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:26.990301 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:26.990571 12834 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:26.990772 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:26.990927 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:26.990993 12440 raft_consensus.cc:2454] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:26.990960 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:26.991119 12832 raft_consensus.cc:2454] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:26.991218 12702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:26.991415 12702 raft_consensus.cc:2454] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:26.991457 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:26.991689 12962 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
W20240502 14:17:26.991612 13030 leader_election.cc:341] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): Illegal state: must be running to vote when last-logged opid is not known
W20240502 14:17:26.991739 12244 leader_election.cc:341] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:26.992177 12245 leader_election.cc:302] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters: af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:26.992421 13031 leader_election.cc:302] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters: d693e089cfa54fbb8df91939410b19db
I20240502 14:17:26.992530 14390 raft_consensus.cc:2781] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:26.992625 14390 raft_consensus.cc:482] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:26.992820 14390 raft_consensus.cc:3037] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:26.992908 14263 raft_consensus.cc:2781] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:26.992882 12306 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:26.993022 14263 raft_consensus.cc:482] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:26.993113 14263 raft_consensus.cc:3037] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
W20240502 14:17:26.993427 13031 leader_election.cc:341] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:26.994067 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:26.995934 14263 raft_consensus.cc:504] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:26.997072 12702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:26.997262 12702 raft_consensus.cc:3037] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:26.997269 14390 raft_consensus.cc:504] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:26.998942 12440 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:26.999174 12440 raft_consensus.cc:3037] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:26.999243 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:26.999505 12571 raft_consensus.cc:3037] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.000012 12702 raft_consensus.cc:2454] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
W20240502 14:17:27.000731 13029 leader_election.cc:341] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:27.002003 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:27.002710 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:27.002914 12441 raft_consensus.cc:3037] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.003038 14263 leader_election.cc:288] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Requested vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:27.003278 14263 raft_consensus.cc:482] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:27.003261 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:27.003564 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
W20240502 14:17:27.003760 13029 leader_election.cc:341] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:27.003794 12832 raft_consensus.cc:3037] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.003422 14263 raft_consensus.cc:504] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:27.005144 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:27.005365 12702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
W20240502 14:17:27.005879 13031 leader_election.cc:341] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:27.006974 14263 leader_election.cc:288] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:27.006930 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:27.007146 12834 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 0.
I20240502 14:17:27.007189 12832 raft_consensus.cc:2454] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:27.007118 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:27.007440 12308 raft_consensus.cc:2379] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:27.007966 12571 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
W20240502 14:17:27.009097 13030 leader_election.cc:341] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:27.009294 12702 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:27.009768 14390 leader_election.cc:288] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 election: Requested vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:27.009713 12439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:27.009817 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:27.009984 12834 raft_consensus.cc:3037] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
W20240502 14:17:27.011724 12244 leader_election.cc:341] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:27.012226 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:27.012456 13096 raft_consensus.cc:3037] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.013347 13031 leader_election.cc:302] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, af704d4ab2f2496a833c692bd3d25fde, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:27.013494 12834 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:27.014309 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:27.014537 12964 raft_consensus.cc:3037] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.015904 13096 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:27.016463 12244 leader_election.cc:302] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters: af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:27.016793 14390 raft_consensus.cc:2781] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:27.017887 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:27.019223 12440 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:27.019897 12441 raft_consensus.cc:2454] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in term 1.
I20240502 14:17:27.020481 13030 leader_election.cc:302] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e; no voters: 53479f0fb09c454e9b3cbe7398bf8874, d693e089cfa54fbb8df91939410b19db
I20240502 14:17:27.021128 14398 raft_consensus.cc:2726] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:27.021399 14398 raft_consensus.cc:2781] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:27.021591 14398 raft_consensus.cc:686] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Becoming Leader. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Running, Role: LEADER
I20240502 14:17:27.021584 12964 raft_consensus.cc:2454] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:27.022015 14398 consensus_queue.cc:237] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.022842 12306 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
W20240502 14:17:27.023556 13031 leader_election.cc:341] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:27.024511 14386 leader_election.cc:288] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:27.025732 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:27.026007 13096 raft_consensus.cc:2362] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1. Re-sending same reply.
I20240502 14:17:27.027523 14390 raft_consensus.cc:686] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Becoming Leader. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Running, Role: LEADER
I20240502 14:17:27.028174 14390 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:27.028934 14369 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:27.029227 14369 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.122s user 0.020s sys 0.020s
I20240502 14:17:27.029320 14373 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:27.029608 14373 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.126s user 0.011s sys 0.028s
I20240502 14:17:27.030381 14373 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.030762 14373 raft_consensus.cc:374] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:27.030609 14369 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.030956 14373 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:27.030961 14369 raft_consensus.cc:374] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:27.031060 14369 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:27.031355 14373 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.031520 14369 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.033493 14369 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 14:17:27.034076 14369 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:27.036304 14369 tablet_bootstrap.cc:654] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:27.043782 12161 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db reported cstate change: term changed from 0 to 1, leader changed from <none> to d693e089cfa54fbb8df91939410b19db (127.27.220.134). New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:27.046321 14373 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.017s user 0.000s sys 0.002s
I20240502 14:17:27.049439 14369 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:27.050127 14369 ts_tablet_manager.cc:1367] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.016s user 0.003s sys 0.007s
I20240502 14:17:27.051277 14369 raft_consensus.cc:348] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.051726 14369 raft_consensus.cc:374] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:27.051939 14369 raft_consensus.cc:729] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:27.052393 14369 consensus_queue.cc:260] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.053357 14373 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:27.056625 14373 tablet_bootstrap.cc:654] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:27.057662 14368 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:27.057888 14368 ts_tablet_manager.cc:1367] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.098s user 0.009s sys 0.012s
I20240502 14:17:27.058876 14368 raft_consensus.cc:348] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.059244 14368 raft_consensus.cc:374] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:27.059360 14368 raft_consensus.cc:729] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:27.059655 14368 consensus_queue.cc:260] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.060564 14368 ts_tablet_manager.cc:1398] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.003s user 0.000s sys 0.000s
I20240502 14:17:27.063022 14369 ts_tablet_manager.cc:1398] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.011s user 0.003s sys 0.000s
I20240502 14:17:27.067632 14373 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:27.067809 14373 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.003s
I20240502 14:17:27.068605 14373 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:27.068878 14373 raft_consensus.cc:374] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:27.068965 14373 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:27.069368 14373 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:27.072400 12162 catalog_manager.cc:5484] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: term changed from 0 to 1, leader changed from <none> to 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135). New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:27.074874 14372 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:27.075199 14372 ts_tablet_manager.cc:1367] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.093s user 0.005s sys 0.000s
I20240502 14:17:27.076467 14373 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.009s user 0.003s sys 0.000s
I20240502 14:17:27.076277 14372 raft_consensus.cc:348] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.076730 14372 raft_consensus.cc:374] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:27.078702 14372 raft_consensus.cc:729] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:27.079118 14372 consensus_queue.cc:260] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.079345 14326 raft_consensus.cc:482] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:27.079530 14326 raft_consensus.cc:504] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.080122 14372 ts_tablet_manager.cc:1398] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 14:17:27.080891 14326 leader_election.cc:288] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:27.080909 12161 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: term changed from 0 to 1, leader changed from <none> to 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:27.081883 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:27.082104 12964 raft_consensus.cc:2454] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:27.082152 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:27.082495 12703 raft_consensus.cc:2379] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:27.082676 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:27.082947 12441 raft_consensus.cc:2379] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:27.083693 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:27.083770 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:27.084043 12834 raft_consensus.cc:2379] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate 84d1fcd2d4cf472db7959f63a7f0ddaf in this term.
I20240502 14:17:27.085954 12507 leader_election.cc:302] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, d693e089cfa54fbb8df91939410b19db; no voters: 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:27.086458 14268 raft_consensus.cc:3037] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.090848 14268 raft_consensus.cc:2726] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:27.092233 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f8526552032e4a45ab4be4743f12abd0" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:27.092510 12308 raft_consensus.cc:2454] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:27.201447 14326 raft_consensus.cc:482] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:27.201697 14326 raft_consensus.cc:504] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:27.202982 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:27.203164 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:27.203382 12834 raft_consensus.cc:2379] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate d693e089cfa54fbb8df91939410b19db in this term.
I20240502 14:17:27.203486 12703 raft_consensus.cc:2454] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:27.204572 14326 leader_election.cc:288] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:27.205665 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:27.205942 12308 raft_consensus.cc:2379] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate d693e089cfa54fbb8df91939410b19db in this term.
I20240502 14:17:27.206622 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:27.207144 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:27.207406 13096 raft_consensus.cc:2379] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate d693e089cfa54fbb8df91939410b19db in this term.
I20240502 14:17:27.208089 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9cd04934b5264d7a9643974c0386cac4" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:27.208359 12441 raft_consensus.cc:2379] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 53479f0fb09c454e9b3cbe7398bf8874 in current term 1: Already voted for candidate d693e089cfa54fbb8df91939410b19db in this term.
I20240502 14:17:27.208804 12506 leader_election.cc:302] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, af704d4ab2f2496a833c692bd3d25fde; no voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:27.209120 14326 raft_consensus.cc:3037] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.211971 14326 raft_consensus.cc:2726] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:27.356034 14252 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.376425 14383 raft_consensus.cc:482] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:27.376616 14383 raft_consensus.cc:504] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:27.378132 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:27.378432 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:27.378783 12441 raft_consensus.cc:2379] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:27.379662 14383 leader_election.cc:288] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:27.380532 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:27.380883 12964 raft_consensus.cc:2379] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:27.381234 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:27.381556 13096 raft_consensus.cc:2379] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:27.381954 12638 leader_election.cc:302] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: af704d4ab2f2496a833c692bd3d25fde; no voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:27.382108 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:27.382267 14383 raft_consensus.cc:3037] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.382377 12572 raft_consensus.cc:2379] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:27.382964 12834 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b6bcd8515d234027abfd01ea5dbb3ede" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:27.383318 12834 raft_consensus.cc:2379] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate af704d4ab2f2496a833c692bd3d25fde in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:27.385205 14383 raft_consensus.cc:2726] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:27.386574 14252 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.406560 12964 raft_consensus.cc:3037] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.410593 14398 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.423666 14252 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.432243 14390 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.440464 12703 raft_consensus.cc:3037] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.445515 14324 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.453747 14324 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.467433 14414 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.475332 14397 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.480592 14397 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.512610 14398 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.518203 14398 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.561007 14423 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.054s
I20240502 14:17:27.568143 14263 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.570047 14263 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:27.603629 12245 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:45069, user_credentials={real_user=slave}} blocked reactor thread for 49134.8us
W20240502 14:17:27.638600 14404 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:27.639158 14404 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:27.656220 14404 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
W20240502 14:17:27.660738 14427 log.cc:924] Time spent T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e: Append to log took a long time: real 0.055s user 0.000s sys 0.000s
W20240502 14:17:27.678998 13031 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:45069, user_credentials={real_user=slave}} blocked reactor thread for 56718.9us
I20240502 14:17:27.690446 12306 raft_consensus.cc:3037] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:27.694130 14426 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.694710 14426 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:27.695420 14443 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_5'
W20240502 14:17:28.142400 12376 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.132:45069, user_credentials={real_user=slave}} blocked reactor thread for 50573.8us
INSERT report
rows total: 2000
time total: 547.385 ms
time per row: 0.273693 ms
I20240502 14:17:28.409009 12164 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:60530:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 14:17:28.409798 12164 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_6 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 14:17:28.422793 12288 tablet_service.cc:1467] Processing CreateTablet for tablet 0397d318dc56416c9d8043f98462aeee (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.423000 12419 tablet_service.cc:1467] Processing CreateTablet for tablet 0397d318dc56416c9d8043f98462aeee (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.423328 12288 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0397d318dc56416c9d8043f98462aeee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.423316 12550 tablet_service.cc:1467] Processing CreateTablet for tablet 0397d318dc56416c9d8043f98462aeee (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.423523 12419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0397d318dc56416c9d8043f98462aeee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.423753 12550 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0397d318dc56416c9d8043f98462aeee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.429132 12813 tablet_service.cc:1467] Processing CreateTablet for tablet 0397d318dc56416c9d8043f98462aeee (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.429143 12682 tablet_service.cc:1467] Processing CreateTablet for tablet 0397d318dc56416c9d8043f98462aeee (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.429208 12943 tablet_service.cc:1467] Processing CreateTablet for tablet 0397d318dc56416c9d8043f98462aeee (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.429612 12682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0397d318dc56416c9d8043f98462aeee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.429625 12813 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0397d318dc56416c9d8043f98462aeee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.429678 12943 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0397d318dc56416c9d8043f98462aeee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.435773 13074 tablet_service.cc:1467] Processing CreateTablet for tablet 0397d318dc56416c9d8043f98462aeee (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.436273 13074 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0397d318dc56416c9d8043f98462aeee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.440713 12944 tablet_service.cc:1467] Processing CreateTablet for tablet ddf70e508caa4f48a9255be579060d97 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.441200 12944 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.441629 12683 tablet_service.cc:1467] Processing CreateTablet for tablet ddf70e508caa4f48a9255be579060d97 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.442046 12683 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.446269 12812 tablet_service.cc:1467] Processing CreateTablet for tablet ddf70e508caa4f48a9255be579060d97 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.446321 13076 tablet_service.cc:1467] Processing CreateTablet for tablet ddf70e508caa4f48a9255be579060d97 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.446727 12812 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.446748 13076 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.447450 12287 tablet_service.cc:1467] Processing CreateTablet for tablet ddf70e508caa4f48a9255be579060d97 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.447897 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.449528 12945 tablet_service.cc:1467] Processing CreateTablet for tablet 29809afc38e54766ba33a4e24311f8f4 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.449939 12945 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.449915 12814 tablet_service.cc:1467] Processing CreateTablet for tablet 29809afc38e54766ba33a4e24311f8f4 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.450335 12814 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.450675 12681 tablet_service.cc:1467] Processing CreateTablet for tablet 29809afc38e54766ba33a4e24311f8f4 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.451115 12681 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.454012 12420 tablet_service.cc:1467] Processing CreateTablet for tablet ddf70e508caa4f48a9255be579060d97 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.454480 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.454464 12552 tablet_service.cc:1467] Processing CreateTablet for tablet ddf70e508caa4f48a9255be579060d97 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.454878 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.471272 13075 tablet_service.cc:1467] Processing CreateTablet for tablet 29809afc38e54766ba33a4e24311f8f4 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.471977 13075 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.473824 12287 tablet_service.cc:1467] Processing CreateTablet for tablet 29809afc38e54766ba33a4e24311f8f4 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.474458 12287 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.474383 12420 tablet_service.cc:1467] Processing CreateTablet for tablet 29809afc38e54766ba33a4e24311f8f4 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.475180 12420 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.484860 12552 tablet_service.cc:1467] Processing CreateTablet for tablet 29809afc38e54766ba33a4e24311f8f4 (DEFAULT_TABLE table=rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 14:17:28.485287 14526 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:28.485574 12552 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:28.489068 14529 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:28.491844 14529 tablet_bootstrap.cc:654] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.493176 14523 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:28.493369 14526 tablet_bootstrap.cc:654] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.494232 14525 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:28.494565 14524 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:28.497179 14523 tablet_bootstrap.cc:654] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.497576 14527 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:28.498167 14528 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:28.499272 14524 tablet_bootstrap.cc:654] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.500836 14529 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:28.501042 14529 ts_tablet_manager.cc:1367] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.000s
I20240502 14:17:28.502063 14527 tablet_bootstrap.cc:654] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.502025 14529 raft_consensus.cc:348] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.502449 14529 raft_consensus.cc:374] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.502571 14529 raft_consensus.cc:729] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.502872 14529 consensus_queue.cc:260] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.503751 14525 tablet_bootstrap.cc:654] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.503753 14529 ts_tablet_manager.cc:1398] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.003s user 0.000s sys 0.000s
I20240502 14:17:28.506124 14523 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:28.506443 14523 ts_tablet_manager.cc:1367] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.013s user 0.006s sys 0.000s
I20240502 14:17:28.506565 14529 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:28.506745 14528 tablet_bootstrap.cc:654] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.507547 14523 raft_consensus.cc:348] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.509546 14526 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:28.509783 14526 ts_tablet_manager.cc:1367] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.025s user 0.004s sys 0.003s
I20240502 14:17:28.510818 14526 raft_consensus.cc:348] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.511255 14526 raft_consensus.cc:374] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.511416 14526 raft_consensus.cc:729] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.511602 14529 tablet_bootstrap.cc:654] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.511848 14526 consensus_queue.cc:260] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.513900 14526 ts_tablet_manager.cc:1398] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.004s user 0.001s sys 0.002s
I20240502 14:17:28.514281 14526 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:28.516357 14523 raft_consensus.cc:374] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.517166 14523 raft_consensus.cc:729] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.517567 14523 consensus_queue.cc:260] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.518307 14528 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:28.518321 14524 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:28.518471 14524 ts_tablet_manager.cc:1367] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.024s user 0.005s sys 0.005s
I20240502 14:17:28.519275 14524 raft_consensus.cc:348] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.519572 14524 raft_consensus.cc:374] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.519658 14524 raft_consensus.cc:729] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.519909 14524 consensus_queue.cc:260] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.520601 14528 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.023s user 0.005s sys 0.000s
I20240502 14:17:28.520705 14525 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:28.521663 14524 ts_tablet_manager.cc:1398] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 14:17:28.521921 14525 ts_tablet_manager.cc:1367] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.028s user 0.002s sys 0.004s
I20240502 14:17:28.522025 14524 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:28.522893 14525 raft_consensus.cc:348] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.521592 14528 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.523541 14528 raft_consensus.cc:374] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.523694 14528 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.523866 14525 raft_consensus.cc:374] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.524219 14523 ts_tablet_manager.cc:1398] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.017s user 0.002s sys 0.000s
I20240502 14:17:28.524379 14525 raft_consensus.cc:729] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.524662 14525 consensus_queue.cc:260] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.524163 14528 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.527724 14526 tablet_bootstrap.cc:654] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.528223 14523 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:28.529070 14528 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
I20240502 14:17:28.529421 14528 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:28.530064 14524 tablet_bootstrap.cc:654] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.530786 14527 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:28.530977 14527 ts_tablet_manager.cc:1367] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.034s user 0.003s sys 0.002s
I20240502 14:17:28.531893 14527 raft_consensus.cc:348] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.532352 14527 raft_consensus.cc:374] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.532477 14527 raft_consensus.cc:729] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.532732 14527 consensus_queue.cc:260] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.533110 14529 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:28.533360 14529 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.027s user 0.003s sys 0.005s
I20240502 14:17:28.533592 14525 ts_tablet_manager.cc:1398] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.011s user 0.000s sys 0.002s
I20240502 14:17:28.533975 14525 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
I20240502 14:17:28.534343 14529 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.534724 14529 raft_consensus.cc:374] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.534924 14523 tablet_bootstrap.cc:654] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.534984 14529 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.535390 14529 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.536132 14528 tablet_bootstrap.cc:654] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.536890 14529 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:28.537195 14529 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Bootstrap starting.
I20240502 14:17:28.537859 14525 tablet_bootstrap.cc:654] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.542464 14529 tablet_bootstrap.cc:654] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.543726 14523 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:28.543951 14523 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.000s
I20240502 14:17:28.544854 14523 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.545195 14523 raft_consensus.cc:374] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.545289 14523 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.545778 14524 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:28.545969 14524 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.024s user 0.003s sys 0.006s
I20240502 14:17:28.546409 14528 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:28.546696 14524 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.547165 14527 ts_tablet_manager.cc:1398] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.016s user 0.004s sys 0.002s
I20240502 14:17:28.547196 14524 raft_consensus.cc:374] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.547294 14524 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.547608 14524 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.548682 14526 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:28.548893 14526 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.035s user 0.002s sys 0.003s
I20240502 14:17:28.550518 14526 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.550993 14526 raft_consensus.cc:374] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.551144 14526 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.551456 14526 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.552143 14526 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:17:28.552179 14524 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 14:17:28.552402 14527 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:28.555698 14529 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: No bootstrap required, opened a new log
I20240502 14:17:28.555867 14529 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent bootstrapping tablet: real 0.019s user 0.005s sys 0.000s
I20240502 14:17:28.556648 14529 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.556504 14523 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.556946 14529 raft_consensus.cc:374] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.557060 14529 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 84d1fcd2d4cf472db7959f63a7f0ddaf, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.557825 14527 tablet_bootstrap.cc:654] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.558189 14528 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.029s user 0.000s sys 0.004s
I20240502 14:17:28.557369 14529 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.559041 14523 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.015s user 0.003s sys 0.000s
I20240502 14:17:28.559902 14529 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
I20240502 14:17:28.560456 14516 raft_consensus.cc:482] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:28.560631 14516 raft_consensus.cc:504] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.561200 14523 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: Bootstrap starting.
I20240502 14:17:28.562096 14516 leader_election.cc:288] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:28.562770 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:28.563014 12832 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:28.563642 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:28.562016 14528 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.564582 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:28.564865 14528 raft_consensus.cc:374] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.564867 13096 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:28.565414 12439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:28.565704 12439 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:28.566417 14525 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:28.566474 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:28.566737 14525 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.033s user 0.000s sys 0.005s
I20240502 14:17:28.566726 12701 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
W20240502 14:17:28.567173 12243 leader_election.cc:341] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:28.567911 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:28.568035 14524 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f: Bootstrap starting.
I20240502 14:17:28.568147 12572 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 0.
I20240502 14:17:28.567890 14525 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.568185 14528 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.568683 14525 raft_consensus.cc:374] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.568938 14528 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.569346 14525 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.568630 12242 leader_election.cc:302] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters: d693e089cfa54fbb8df91939410b19db
I20240502 14:17:28.569880 14524 tablet_bootstrap.cc:654] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.569890 14516 raft_consensus.cc:2781] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:28.569999 14516 raft_consensus.cc:482] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:28.570099 14516 raft_consensus.cc:3037] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.570601 14528 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.011s user 0.001s sys 0.001s
I20240502 14:17:28.570711 14525 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.572929 14516 raft_consensus.cc:504] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.576077 12439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:28.576355 12439 raft_consensus.cc:3037] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.577180 14525 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.010s user 0.000s sys 0.002s
I20240502 14:17:28.577630 14528 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:28.571630 14523 tablet_bootstrap.cc:654] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.577800 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:28.578050 12832 raft_consensus.cc:3037] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.578176 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:28.578451 12572 raft_consensus.cc:3037] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.581686 12832 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:28.583654 14516 leader_election.cc:288] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 election: Requested vote from peers eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:28.567909 14526 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:28.584026 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:28.584758 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:28.585076 13096 raft_consensus.cc:3037] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.585644 14525 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e: Bootstrap starting.
W20240502 14:17:28.586032 12243 leader_election.cc:341] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:28.586385 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "70a425a77fe54cbcb9755f9c540fd367" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:28.586712 12701 raft_consensus.cc:3037] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.587759 14526 tablet_bootstrap.cc:654] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.589154 14528 tablet_bootstrap.cc:654] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.589771 14525 tablet_bootstrap.cc:654] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.591679 13096 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:28.592555 14524 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f: No bootstrap required, opened a new log
I20240502 14:17:28.592751 14524 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent bootstrapping tablet: real 0.025s user 0.012s sys 0.007s
I20240502 14:17:28.593199 12701 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:28.593752 12244 leader_election.cc:302] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f; no voters: d693e089cfa54fbb8df91939410b19db
I20240502 14:17:28.593755 14524 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.594120 14524 raft_consensus.cc:374] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.594233 14524 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c78bd31d8d2142bf9d8e06b5a0074e0f, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.594537 14524 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.595338 14524 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:28.596838 12439 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:28.597571 14516 raft_consensus.cc:2781] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:28.597724 14516 raft_consensus.cc:686] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Becoming Leader. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Running, Role: LEADER
I20240502 14:17:28.598126 14516 consensus_queue.cc:237] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.599031 14526 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367: No bootstrap required, opened a new log
I20240502 14:17:28.599506 14526 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.032s user 0.001s sys 0.004s
I20240502 14:17:28.600623 14527 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:28.600620 14526 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.600891 14526 raft_consensus.cc:374] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.600946 14527 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.053s user 0.003s sys 0.004s
I20240502 14:17:28.601002 14526 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.601352 14268 raft_consensus.cc:482] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:28.601274 14526 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.601936 14526 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:28.602047 14527 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.602317 14527 raft_consensus.cc:374] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.602408 14527 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.602664 14527 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.601868 14268 raft_consensus.cc:504] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.599066 14528 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db: No bootstrap required, opened a new log
I20240502 14:17:28.603435 14528 ts_tablet_manager.cc:1367] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.026s user 0.002s sys 0.003s
I20240502 14:17:28.604101 14268 leader_election.cc:288] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:28.604436 14527 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:28.604638 14528 raft_consensus.cc:348] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.605015 14528 raft_consensus.cc:374] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.605162 14528 raft_consensus.cc:729] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.605577 14527 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:28.605476 14528 consensus_queue.cc:260] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.605073 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:28.606328 12964 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:28.607111 14527 tablet_bootstrap.cc:654] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde: Neither blocks nor log segments found. Creating new log.
I20240502 14:17:28.607585 12439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:28.607825 12439 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:28.608423 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:28.608657 13096 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:28.609282 14528 ts_tablet_manager.cc:1398] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 14:17:28.609664 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:28.609913 12309 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:28.610380 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:28.610615 12832 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 0.
I20240502 14:17:28.610823 14525 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e: No bootstrap required, opened a new log
I20240502 14:17:28.611027 14525 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e: Time spent bootstrapping tablet: real 0.026s user 0.005s sys 0.000s
I20240502 14:17:28.610931 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:28.612412 12506 leader_election.cc:302] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, 84d1fcd2d4cf472db7959f63a7f0ddaf, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:28.613142 14545 raft_consensus.cc:2781] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:28.613255 14545 raft_consensus.cc:482] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:28.613613 14545 raft_consensus.cc:3037] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.612147 14525 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.614197 14525 raft_consensus.cc:374] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20240502 14:17:28.615419 12507 leader_election.cc:341] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:28.615526 12160 catalog_manager.cc:5484] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: term changed from 0 to 1, leader changed from <none> to 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129). New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:28.617237 12572 raft_consensus.cc:2454] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 70a425a77fe54cbcb9755f9c540fd367 in term 1.
I20240502 14:17:28.619522 14527 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde: No bootstrap required, opened a new log
I20240502 14:17:28.619267 14545 raft_consensus.cc:504] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.619727 14527 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.006s
I20240502 14:17:28.620640 14527 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.620985 14527 raft_consensus.cc:374] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.621119 14527 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.621196 12964 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db"
I20240502 14:17:28.621428 12964 raft_consensus.cc:3037] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.621600 12439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:28.621816 12439 raft_consensus.cc:3037] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.614305 14525 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eddeff583cf34c8dab2942fb36abbf9e, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.621639 14527 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.622046 14539 raft_consensus.cc:482] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:28.621984 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:28.622191 12832 raft_consensus.cc:3037] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.622181 14539 raft_consensus.cc:504] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.622195 14525 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.623363 14539 leader_election.cc:288] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:28.624665 12441 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:28.624840 12439 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:28.625077 12833 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:28.625690 12962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d693e089cfa54fbb8df91939410b19db" is_pre_election: true
I20240502 14:17:28.625780 12832 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:28.626607 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:28.626685 14545 leader_election.cc:288] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Requested vote from peers d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:28.626865 12309 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af704d4ab2f2496a833c692bd3d25fde in term 0.
I20240502 14:17:28.627228 14525 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e: Time spent starting tablet: real 0.016s user 0.002s sys 0.000s
I20240502 14:17:28.627628 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:28.627910 13096 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate af704d4ab2f2496a833c692bd3d25fde in term 0.
I20240502 14:17:28.629105 14527 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240502 14:17:28.630704 12964 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:28.631187 12506 leader_election.cc:302] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 53479f0fb09c454e9b3cbe7398bf8874, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:28.631541 14545 raft_consensus.cc:2781] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:28.631525 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:28.631690 12701 raft_consensus.cc:3037] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.631727 14545 raft_consensus.cc:686] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Becoming Leader. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Running, Role: LEADER
I20240502 14:17:28.631983 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:28.632200 13096 raft_consensus.cc:3037] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.632237 12308 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "53479f0fb09c454e9b3cbe7398bf8874" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:28.632062 14545 consensus_queue.cc:237] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.632442 12308 raft_consensus.cc:3037] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.634311 14482 raft_consensus.cc:482] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:28.634508 14482 raft_consensus.cc:504] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.635219 13096 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:28.635694 12308 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:28.635818 14482 leader_election.cc:288] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:28.636229 12439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:28.636482 12439 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:28.637586 12703 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:28.637811 12703 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:28.637784 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:28.638067 12309 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:28.638681 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:28.638873 12701 raft_consensus.cc:2454] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 53479f0fb09c454e9b3cbe7398bf8874 in term 1.
I20240502 14:17:28.638911 13096 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:28.639310 12900 leader_election.cc:302] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:28.639678 14482 raft_consensus.cc:2781] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 14:17:28.639796 14482 raft_consensus.cc:482] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 14:17:28.639892 14482 raft_consensus.cc:3037] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.640208 12571 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
W20240502 14:17:28.641566 12898 leader_election.cc:341] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): Illegal state: must be running to vote when last-logged opid is not known
I20240502 14:17:28.641968 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ddf70e508caa4f48a9255be579060d97" candidate_uuid: "af704d4ab2f2496a833c692bd3d25fde" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:28.642629 14482 raft_consensus.cc:504] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.642872 12636 leader_election.cc:302] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 7 responses out of 7 voters: 3 yes votes; 4 no votes. yes voters: 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, af704d4ab2f2496a833c692bd3d25fde; no voters: 53479f0fb09c454e9b3cbe7398bf8874, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:28.643143 14539 raft_consensus.cc:2726] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:28.643515 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:28.643795 12832 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 0.
I20240502 14:17:28.645216 14523 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: No bootstrap required, opened a new log
I20240502 14:17:28.645392 14523 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent bootstrapping tablet: real 0.084s user 0.006s sys 0.000s
I20240502 14:17:28.645588 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367"
I20240502 14:17:28.645805 12309 raft_consensus.cc:3037] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.646319 14523 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.646584 14523 raft_consensus.cc:374] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 14:17:28.646672 14523 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 53479f0fb09c454e9b3cbe7398bf8874, State: Initialized, Role: FOLLOWER
I20240502 14:17:28.646975 14523 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.647642 14523 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 14:17:28.648602 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde"
I20240502 14:17:28.649344 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f"
I20240502 14:17:28.649603 12832 raft_consensus.cc:3037] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.649765 12439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e"
I20240502 14:17:28.649978 12439 raft_consensus.cc:3037] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.650669 12163 catalog_manager.cc:5484] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: term changed from 0 to 1, leader changed from <none> to 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131). New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:28.651778 12439 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:28.652220 12832 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:28.652510 12701 raft_consensus.cc:3037] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.652868 14482 leader_election.cc:288] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Requested vote from peers c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203), 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:28.653179 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf"
I20240502 14:17:28.653430 13096 raft_consensus.cc:3037] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.654525 12309 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:28.655009 12701 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:28.655411 12900 leader_election.cc:302] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: af704d4ab2f2496a833c692bd3d25fde, c78bd31d8d2142bf9d8e06b5a0074e0f, d693e089cfa54fbb8df91939410b19db, eddeff583cf34c8dab2942fb36abbf9e; no voters:
I20240502 14:17:28.656113 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "29809afc38e54766ba33a4e24311f8f4" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874"
I20240502 14:17:28.656312 12572 raft_consensus.cc:3037] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.657723 14482 raft_consensus.cc:2781] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader election won for term 1
I20240502 14:17:28.659014 13096 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:28.659962 12572 raft_consensus.cc:2454] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d693e089cfa54fbb8df91939410b19db in term 1.
I20240502 14:17:28.665114 14482 raft_consensus.cc:686] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Becoming Leader. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Running, Role: LEADER
I20240502 14:17:28.665469 14482 consensus_queue.cc:237] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:28.669864 12163 catalog_manager.cc:5484] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db reported cstate change: term changed from 0 to 1, leader changed from <none> to d693e089cfa54fbb8df91939410b19db (127.27.220.134). New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:28.706816 14482 raft_consensus.cc:482] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 14:17:28.707053 14482 raft_consensus.cc:504] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:28.708493 12309 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "70a425a77fe54cbcb9755f9c540fd367" is_pre_election: true
I20240502 14:17:28.708550 12439 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eddeff583cf34c8dab2942fb36abbf9e" is_pre_election: true
I20240502 14:17:28.708772 14482 leader_election.cc:288] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613), eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491), 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727), c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069), af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069), 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:28.708874 12439 raft_consensus.cc:2379] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:28.709285 12572 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "53479f0fb09c454e9b3cbe7398bf8874" is_pre_election: true
I20240502 14:17:28.709640 12572 raft_consensus.cc:2379] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:28.709812 13096 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" is_pre_election: true
I20240502 14:17:28.710084 13096 raft_consensus.cc:2379] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:28.710206 12701 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "af704d4ab2f2496a833c692bd3d25fde" is_pre_election: true
I20240502 14:17:28.710500 12701 raft_consensus.cc:2379] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:28.710557 12900 leader_election.cc:302] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: d693e089cfa54fbb8df91939410b19db; no voters: 53479f0fb09c454e9b3cbe7398bf8874, 70a425a77fe54cbcb9755f9c540fd367, 84d1fcd2d4cf472db7959f63a7f0ddaf, eddeff583cf34c8dab2942fb36abbf9e
I20240502 14:17:28.710690 12832 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0397d318dc56416c9d8043f98462aeee" candidate_uuid: "d693e089cfa54fbb8df91939410b19db" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" is_pre_election: true
I20240502 14:17:28.711024 12832 raft_consensus.cc:2379] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate d693e089cfa54fbb8df91939410b19db in current term 1: Already voted for candidate 70a425a77fe54cbcb9755f9c540fd367 in this term.
I20240502 14:17:28.711148 14482 raft_consensus.cc:3037] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 0 FOLLOWER]: Advancing to term 1
I20240502 14:17:28.713886 14482 raft_consensus.cc:2726] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 14:17:29.011754 14516 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.020417 14545 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.050737 14516 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.057235 14516 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.062709 14426 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:29.064102 14558 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:29.064644 14558 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 14:17:29.071934 14567 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.079310 14482 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:29.080811 14558 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 14:17:29.083217 14268 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.086246 14482 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.104925 14482 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.115408 14268 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 14:17:29.125188 12244 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:44203, user_credentials={real_user=slave}} blocked reactor thread for 62856.5us
I20240502 14:17:29.130252 14555 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.122222 14570 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.131390 14555 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.145653 14555 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.147234 14545 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.191282 14545 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 14:17:29.208633 14545 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_6'
INSERT report
rows total: 2000
time total: 494.274 ms
time per row: 0.247137 ms
I20240502 14:17:29.896723 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.136:41465
--local_ip_for_outbound_sockets=127.27.220.136
--tserver_master_addrs=127.27.220.190:36663
--webserver_port=33735
--webserver_interface=127.27.220.136
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 14:17:30.273483 14661 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:30.273805 14661 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:30.273960 14661 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:30.282253 14661 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:30.282378 14661 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:30.282544 14661 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.136
I20240502 14:17:30.292802 14661 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.136:41465
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.27.220.136
--webserver_port=33735
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.136
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:30.294402 14661 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:30.296908 14661 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:30.317276 14667 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:30.318099 14670 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:30.322580 14668 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:30.323575 14661 server_base.cc:1034] running on GCE node
I20240502 14:17:30.324224 14661 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:30.325434 14661 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:30.326741 14661 hybrid_clock.cc:648] HybridClock initialized: now 1714659450326641 us; error 75 us; skew 500 ppm
I20240502 14:17:30.327060 14661 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:30.329291 14661 webserver.cc:460] Webserver started at http://127.27.220.136:33735/ using document root <none> and password file <none>
I20240502 14:17:30.330118 14661 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:30.330243 14661 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:30.335959 14661 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.005s sys 0.000s
I20240502 14:17:30.338790 14676 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:30.340250 14661 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 14:17:30.340441 14661 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "4c04a4a8d8864267be7f67c2abe98f84"
format_stamp: "Formatted at 2024-05-02 14:17:12 on dist-test-slave-j72s"
I20240502 14:17:30.341621 14661 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:30.358986 14661 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:30.360008 14661 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:30.360356 14661 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:30.361529 14661 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:30.363595 14661 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:30.363694 14661 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:30.363867 14661 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:30.363938 14661 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:30.460932 14661 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.136:41465
I20240502 14:17:30.460989 14795 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.136:41465 every 8 connection(s)
I20240502 14:17:30.462671 14661 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 14:17:30.467875 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 14661
I20240502 14:17:30.468652 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.138:33073
--local_ip_for_outbound_sockets=127.27.220.138
--tserver_master_addrs=127.27.220.190:36663
--webserver_port=40151
--webserver_interface=127.27.220.138
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:30.482131 14796 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:30.482475 14796 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:30.483592 14796 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:30.485345 12160 ts_manager.cc:194] Re-registered known tserver with Master: 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465)
I20240502 14:17:30.486339 12160 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.136:47865
W20240502 14:17:30.844709 14800 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:30.845012 14800 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:30.845098 14800 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:30.853555 14800 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:30.853675 14800 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:30.853819 14800 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.138
I20240502 14:17:30.862619 14800 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.138:33073
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--webserver_interface=127.27.220.138
--webserver_port=40151
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.138
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:30.863889 14800 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:30.865784 14800 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:30.878667 14808 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:30.879194 14809 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:30.883777 14811 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:30.884420 14800 server_base.cc:1034] running on GCE node
I20240502 14:17:30.885082 14800 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:30.886174 14800 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:30.887393 14800 hybrid_clock.cc:648] HybridClock initialized: now 1714659450887341 us; error 47 us; skew 500 ppm
I20240502 14:17:30.887724 14800 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:30.890267 14800 webserver.cc:460] Webserver started at http://127.27.220.138:40151/ using document root <none> and password file <none>
I20240502 14:17:30.890959 14800 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:30.891088 14800 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:30.896157 14800 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 14:17:30.899276 14817 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:30.900529 14800 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240502 14:17:30.900673 14800 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
uuid: "1ba6a238e6ad4e4a84eff746ff336144"
format_stamp: "Formatted at 2024-05-02 14:17:13 on dist-test-slave-j72s"
I20240502 14:17:30.901439 14800 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:30.931946 14800 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:30.933015 14800 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:30.933324 14800 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:30.934636 14800 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:30.936532 14800 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:30.936625 14800 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:30.936749 14800 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:30.936800 14800 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:30.987568 14800 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.138:33073
I20240502 14:17:30.987618 14929 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.138:33073 every 8 connection(s)
I20240502 14:17:30.989161 14800 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-9/data/info.pb
I20240502 14:17:30.998508 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 14800
I20240502 14:17:30.999259 28530 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
/tmp/dist-test-taskhB7nWc/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.27.220.137:40043
--local_ip_for_outbound_sockets=127.27.220.137
--tserver_master_addrs=127.27.220.190:36663
--webserver_port=38207
--webserver_interface=127.27.220.137
--builtin_ntp_servers=127.27.220.148:39769
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 14:17:31.002331 14930 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:31.002614 14930 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:31.003418 14930 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:31.011082 12160 ts_manager.cc:194] Re-registered known tserver with Master: 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073)
I20240502 14:17:31.011997 12160 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.138:42105
W20240502 14:17:31.368968 14934 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 14:17:31.369252 14934 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 14:17:31.369419 14934 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 14:17:31.376873 14934 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 14:17:31.376991 14934 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 14:17:31.377141 14934 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.27.220.137
I20240502 14:17:31.387221 14934 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.27.220.148:39769
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.27.220.137:40043
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.27.220.137
--webserver_port=38207
--tserver_master_addrs=127.27.220.190:36663
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.27.220.137
--log_dir=/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision a6923a82b532b3d5c86547dfd9b0772bc97ae884
build type FASTDEBUG
built by None at 02 May 2024 13:43:20 UTC on efd1337a2aa3
build id 91
ASAN enabled
I20240502 14:17:31.388634 14934 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 14:17:31.390333 14934 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 14:17:31.402797 14939 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:31.403306 14940 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 14:17:31.403990 14942 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 14:17:31.404728 14934 server_base.cc:1034] running on GCE node
I20240502 14:17:31.405269 14934 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 14:17:31.406133 14934 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 14:17:31.407323 14934 hybrid_clock.cc:648] HybridClock initialized: now 1714659451407258 us; error 57 us; skew 500 ppm
I20240502 14:17:31.407652 14934 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 14:17:31.409798 14934 webserver.cc:460] Webserver started at http://127.27.220.137:38207/ using document root <none> and password file <none>
I20240502 14:17:31.410506 14934 fs_manager.cc:359] Metadata directory not provided
I20240502 14:17:31.410648 14934 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 14:17:31.415864 14934 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.002s
I20240502 14:17:31.418488 14948 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:31.419755 14934 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 14:17:31.419890 14934 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "95e58ca33b4048e2b000dfbf2b5a0186"
format_stamp: "Formatted at 2024-05-02 14:17:13 on dist-test-slave-j72s"
I20240502 14:17:31.420626 14934 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 14:17:31.443076 14934 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 14:17:31.443923 14934 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 14:17:31.444223 14934 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 14:17:31.445407 14934 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 14:17:31.447211 14934 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 14:17:31.447292 14934 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:31.447495 14934 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 14:17:31.447603 14934 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 14:17:31.484513 14934 rpc_server.cc:307] RPC server started. Bound to: 127.27.220.137:40043
I20240502 14:17:31.484540 15060 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.27.220.137:40043 every 8 connection(s)
I20240502 14:17:31.486013 14934 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240502 14:17:31.489742 14796 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
I20240502 14:17:31.494222 28530 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskhB7nWc/build/asan/bin/kudu as pid 14934
I20240502 14:17:31.499055 15061 heartbeater.cc:347] Connected to a master server at 127.27.220.190:36663
I20240502 14:17:31.499348 15061 heartbeater.cc:464] Registering TS with master...
I20240502 14:17:31.500360 15061 heartbeater.cc:511] Master 127.27.220.190:36663 requested a full tablet report, sending...
I20240502 14:17:31.501802 12160 ts_manager.cc:194] Re-registered known tserver with Master: 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043)
I20240502 14:17:31.502789 12160 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.27.220.137:39879
I20240502 14:17:32.014894 14930 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
W20240502 14:17:32.056996 12747 acceptor_pool.cc:285] unable to collect diagnostics on RPC socket 127.27.220.132:45069: Invalid argument: socket address is ambiguous
I20240502 14:17:32.065609 14865 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:32.067078 12552 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:17:32.082257 14996 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:32.094727 14724 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:32.117027 12287 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:32.149741 12682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:32.171283 13074 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:32.173544 12813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20240502 14:17:32.174093 12508 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:45069, user_credentials={real_user=slave}} blocked reactor thread for 62661.4us
I20240502 14:17:32.229074 12944 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:32.269245 12419 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:17:32.508939 15061 heartbeater.cc:503] Master 127.27.220.190:36663 was elected leader, sending a full tablet report...
I20240502 14:17:32.571326 12552 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:17:32.595158 14996 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:32.595865 14865 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:32.627049 12813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:32.630479 14724 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:32.641273 12682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:32.652748 12944 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:32.661738 12287 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:32.694736 13074 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:32.705142 12419 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:17:33.038451 14865 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:33.069523 12552 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:17:33.078990 14996 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:33.108160 12813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:33.114498 13074 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:33.118580 12944 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:33.119132 14724 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:33.142498 12287 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:33.142581 12419 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:17:33.151054 12682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:33.319891 12834 consensus_queue.cc:237] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:33.331228 12571 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.331447 12309 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.332458 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.333211 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.363808 12701 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.364423 12964 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.364549 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.365375 12440 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.366171 13093 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.366154 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.366832 15216 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.367349 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.382681 15216 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.383539 12702 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.397987 12441 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.403748 12964 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.405162 12166 catalog_manager.cc:5484] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.415496 13096 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.438323 12571 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.444514 12309 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.454813 12834 consensus_queue.cc:237] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:33.458359 12702 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.458927 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.470371 13093 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.470942 12309 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.471081 12571 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.471709 15216 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.472483 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.472944 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.477627 15230 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.479700 13093 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.482266 12308 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.491770 12571 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.492499 12155 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet b33c5e313e784b97987eb07b04f615f8 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:33.495859 12702 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.500553 12964 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.500169 12162 catalog_manager.cc:5484] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.501825 12441 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.507205 15230 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.507653 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.509548 12441 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.509793 12964 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
W20240502 14:17:33.511241 12768 consensus_peers.cc:485] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b33c5e313e784b97987eb07b04f615f8. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:33.519620 12571 consensus_queue.cc:237] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:33.523473 13093 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.523810 12834 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.524950 14659 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.525430 12308 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.526199 14659 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.526436 12702 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.528192 14659 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.528641 14659 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.535827 12441 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.536834 12964 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.537793 15113 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.538403 15113 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.542635 14659 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.546154 12964 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.551935 12441 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.553580 13093 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.554391 12701 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.556455 12308 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.559129 12164 catalog_manager.cc:5484] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.565560 12832 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.575582 12571 consensus_queue.cc:237] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:33.579394 12701 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.579726 13096 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.579914 12832 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.580300 15204 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.580577 14659 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.582538 12308 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.582952 12441 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.581389 15113 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.002s
I20240502 14:17:33.583618 12964 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.585791 15204 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.586275 15204 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.589128 15249 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.589742 13096 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:33.590063 15204 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
W20240502 14:17:33.591104 12506 consensus_peers.cc:485] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 5614c0ba0e374af1853bd2fa586d3a76. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:33.594038 12964 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.596060 12441 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.597849 12152 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 5614c0ba0e374af1853bd2fa586d3a76 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:33.599221 13093 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.601049 12701 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.606318 12166 catalog_manager.cc:5484] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.611781 12308 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.614537 12832 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.628571 12701 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.629108 14263 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.629477 12441 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.630241 14263 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.630704 12571 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.631826 12832 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.633687 12964 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.633818 15153 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.634508 12308 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.634408 15253 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.003s
I20240502 14:17:33.638816 15252 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.639250 15153 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.653285 15153 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.655692 12308 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.660099 12701 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.660441 12571 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.666100 12166 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.670012 12441 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.682718 12964 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.685062 13096 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:33.691790 12832 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.695094 12571 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.695680 12834 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.696568 15253 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.697108 12440 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.697724 15261 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.699419 12964 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.701303 12308 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.702651 15253 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.702975 12701 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.705219 15267 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.706216 15253 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.708788 15261 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.709465 12571 consensus_queue.cc:237] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:33.710582 15261 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.711812 12834 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
W20240502 14:17:33.713471 13030 consensus_peers.cc:485] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: df48755615874edab65577e92249fc78. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:33.714749 12308 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.718631 12702 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.719763 12165 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.726898 12154 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet df48755615874edab65577e92249fc78 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:33.727900 12440 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.728408 12572 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.727387 12964 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.746167 12308 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.747049 12834 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.747020 15249 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.747578 12702 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.748005 14659 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.748216 12440 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.748381 15204 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.749586 12962 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.750234 15249 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.750308 13096 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.751354 15204 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.752529 14659 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.762560 15249 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.763104 12834 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.767607 12309 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.772938 12701 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.776255 12167 catalog_manager.cc:5484] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:33.784330 12571 consensus_queue.cc:237] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:33.790086 12832 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.790809 15249 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 14:17:33.791528 12506 consensus_peers.cc:485] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ddf70e508caa4f48a9255be579060d97. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:33.793120 12701 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.793143 12308 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.794145 14659 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.794236 12440 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.794647 14659 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.795010 15249 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.797186 12962 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.797813 14659 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.798468 12440 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.798838 12702 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.804096 12167 catalog_manager.cc:5484] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.808281 12152 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet ddf70e508caa4f48a9255be579060d97 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:33.809052 15236 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.810506 13096 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.811050 12962 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.811153 12832 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.815708 15236 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.813807 12309 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.817251 12962 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.817601 13093 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:33.824563 13093 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.845832 12441 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.857992 12441 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:33.862049 12702 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.862227 12832 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.862344 12962 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.862761 15269 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.863143 15272 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.863788 15271 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.864464 12571 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.864655 13093 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.865192 15271 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.865748 15271 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.882967 12309 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:33.883762 15269 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:33.892935 15271 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.897416 12571 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.898051 12832 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.900657 12702 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.904435 12167 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.907922 13093 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.914450 12964 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.916813 12309 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:33.924232 15295 ts_tablet_manager.cc:902] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069)
I20240502 14:17:33.940042 12441 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:33.952603 13093 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.953194 12571 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.953255 12309 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.953701 12702 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.953816 15286 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.954037 12832 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.954480 15286 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.954975 15269 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.955636 15269 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.956561 15271 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:33.958524 15295 tablet_copy_client.cc:308] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:45069
I20240502 14:17:33.960736 15286 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.962742 12702 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.965845 12832 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.966071 13093 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.966739 12153 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3b3082d181004353bdff4414cd264d7c with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:33.968369 12571 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.970304 12309 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.971534 12160 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:33.995473 12962 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:33.997957 15271 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:34.000579 12962 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.018510 15306 ts_tablet_manager.cc:902] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:34.022591 12441 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:34.028097 12571 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:34.028970 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 14:17:34.031450 12375 consensus_peers.cc:485] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3b3082d181004353bdff4414cd264d7c. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:34.033586 12854 tablet_copy_service.cc:140] P c78bd31d8d2142bf9d8e06b5a0074e0f: Received BeginTabletCopySession request for tablet b33c5e313e784b97987eb07b04f615f8 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:49687)
I20240502 14:17:34.033835 12854 tablet_copy_service.cc:161] P c78bd31d8d2142bf9d8e06b5a0074e0f: Beginning new tablet copy session on tablet b33c5e313e784b97987eb07b04f615f8 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:49687: session id = 1ba6a238e6ad4e4a84eff746ff336144-b33c5e313e784b97987eb07b04f615f8
I20240502 14:17:34.035290 13093 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:34.035312 12832 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:34.035871 15286 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:34.036550 15272 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:34.041061 12854 tablet_copy_source_session.cc:205] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:34.042937 12702 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:34.045034 12309 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:34.045702 15272 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:34.046203 15272 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:34.046976 15295 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:34.049873 12962 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:34.050447 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:34.053766 15272 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:34.057919 12701 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:34.066704 12571 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:34.069916 12306 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:34.072017 15295 tablet_copy_client.cc:783] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:34.072417 15295 tablet_copy_client.cc:647] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:34.076085 12167 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:34.076798 15295 tablet_copy_client.cc:523] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:34.078755 13096 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:34.081457 15306 tablet_copy_client.cc:308] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:34.082392 12964 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:34.085342 15295 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:34.105142 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet 5614c0ba0e374af1853bd2fa586d3a76 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:39309)
I20240502 14:17:34.105470 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet 5614c0ba0e374af1853bd2fa586d3a76 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:39309: session id = 1ba6a238e6ad4e4a84eff746ff336144-5614c0ba0e374af1853bd2fa586d3a76
I20240502 14:17:34.111438 12592 tablet_copy_source_session.cc:205] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:34.112804 12439 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.113760 12832 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:34.118438 15295 log.cc:824] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:34.119272 15306 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 5614c0ba0e374af1853bd2fa586d3a76. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:34.125420 12702 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:34.127575 12964 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 14:17:34.127729 12375 consensus_peers.cc:485] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c9ff45109a6d4dc9aba4617819929981. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:34.129870 12306 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:34.130519 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:34.132270 13093 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:34.132251 12571 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:34.132840 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:34.133428 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:34.133770 15272 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:34.135524 15269 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:34.143051 15306 tablet_copy_client.cc:783] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:34.142266 15271 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.143443 15306 tablet_copy_client.cc:647] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:34.148576 15306 tablet_copy_client.cc:523] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:34.154426 15325 ts_tablet_manager.cc:902] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:34.154539 12306 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.156082 15306 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:34.161866 12964 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.164400 13093 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.165997 12834 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:34.168017 15092 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:34.169535 12153 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet c9ff45109a6d4dc9aba4617819929981 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:34.171715 12702 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.174782 12167 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.202870 15325 tablet_copy_client.cc:308] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:34.203769 12572 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.209079 12832 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:34.239542 13116 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet df48755615874edab65577e92249fc78 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:60037)
I20240502 14:17:34.239787 13116 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet df48755615874edab65577e92249fc78 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:60037: session id = 1ba6a238e6ad4e4a84eff746ff336144-df48755615874edab65577e92249fc78
I20240502 14:17:34.243646 13116 tablet_copy_source_session.cc:205] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:34.245898 15325 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:34.253561 15325 tablet_copy_client.cc:783] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:34.253854 15325 tablet_copy_client.cc:647] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:34.259191 15306 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:34.259742 15306 tablet_bootstrap.cc:492] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:34.260300 15306 ts_tablet_manager.cc:1367] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.104s user 0.052s sys 0.031s
I20240502 14:17:34.262378 15325 tablet_copy_client.cc:523] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:34.267481 15306 raft_consensus.cc:348] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.268085 15306 raft_consensus.cc:729] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:34.268604 15325 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:34.268707 15306 consensus_queue.cc:260] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.277050 15295 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:34.277607 15295 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:34.279479 15295 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.194s user 0.067s sys 0.042s
I20240502 14:17:34.280546 15295 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.280887 15295 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:34.281286 15306 ts_tablet_manager.cc:1398] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.021s user 0.012s sys 0.004s
I20240502 14:17:34.281122 15295 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.281773 15295 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:17:34.282687 12854 tablet_copy_service.cc:342] P c78bd31d8d2142bf9d8e06b5a0074e0f: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-b33c5e313e784b97987eb07b04f615f8 received from {username='slave'} at 127.27.220.138:49687
I20240502 14:17:34.282881 12854 tablet_copy_service.cc:433] P c78bd31d8d2142bf9d8e06b5a0074e0f: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-b33c5e313e784b97987eb07b04f615f8 on tablet b33c5e313e784b97987eb07b04f615f8 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:34.284570 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-5614c0ba0e374af1853bd2fa586d3a76 received from {username='slave'} at 127.27.220.138:39309
I20240502 14:17:34.284775 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-5614c0ba0e374af1853bd2fa586d3a76 on tablet 5614c0ba0e374af1853bd2fa586d3a76 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:34.311479 15295 ts_tablet_manager.cc:902] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:34.331455 15325 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:34.333237 15325 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:34.333824 15325 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.065s user 0.047s sys 0.004s
I20240502 14:17:34.335153 15325 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.335799 15325 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:34.336149 15325 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.337078 15325 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:34.338160 13116 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-df48755615874edab65577e92249fc78 received from {username='slave'} at 127.27.220.138:60037
I20240502 14:17:34.338330 13116 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-df48755615874edab65577e92249fc78 on tablet df48755615874edab65577e92249fc78 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:34.346529 15295 tablet_copy_client.cc:308] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:34.347173 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet ddf70e508caa4f48a9255be579060d97 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:39309)
I20240502 14:17:34.347362 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet ddf70e508caa4f48a9255be579060d97 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:39309: session id = 1ba6a238e6ad4e4a84eff746ff336144-ddf70e508caa4f48a9255be579060d97
I20240502 14:17:34.350262 12592 tablet_copy_source_session.cc:205] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:34.351471 15295 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:34.357235 15295 tablet_copy_client.cc:783] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:34.357467 15295 tablet_copy_client.cc:647] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:34.358917 15295 tablet_copy_client.cc:523] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:34.361115 15295 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:34.405459 15295 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:34.405920 15295 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:34.406239 15295 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.045s user 0.040s sys 0.004s
I20240502 14:17:34.406975 15295 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.407327 15295 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:34.407546 15295 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.408217 15295 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:34.408900 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-ddf70e508caa4f48a9255be579060d97 received from {username='slave'} at 127.27.220.138:39309
I20240502 14:17:34.409035 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-ddf70e508caa4f48a9255be579060d97 on tablet ddf70e508caa4f48a9255be579060d97 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:34.426203 14885 raft_consensus.cc:1203] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:34.530983 15295 ts_tablet_manager.cc:902] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:34.532255 15295 tablet_copy_client.cc:308] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:34491
I20240502 14:17:34.534443 14885 raft_consensus.cc:1203] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:34.539142 12461 tablet_copy_service.cc:140] P eddeff583cf34c8dab2942fb36abbf9e: Received BeginTabletCopySession request for tablet 3b3082d181004353bdff4414cd264d7c from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:55391)
I20240502 14:17:34.539327 12461 tablet_copy_service.cc:161] P eddeff583cf34c8dab2942fb36abbf9e: Beginning new tablet copy session on tablet 3b3082d181004353bdff4414cd264d7c from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:55391: session id = 1ba6a238e6ad4e4a84eff746ff336144-3b3082d181004353bdff4414cd264d7c
I20240502 14:17:34.542153 12461 tablet_copy_source_session.cc:205] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:34.543628 15295 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:34.548344 15295 tablet_copy_client.cc:783] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:34.548558 15295 tablet_copy_client.cc:647] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:34.550307 15295 tablet_copy_client.cc:523] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:34.552902 15295 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:34.584889 15295 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:34.586598 15295 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:34.586997 15295 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.034s user 0.034s sys 0.001s
I20240502 14:17:34.587854 15295 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.588197 15295 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:34.588583 15295 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.589273 15295 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240502 14:17:34.590117 12461 tablet_copy_service.cc:342] P eddeff583cf34c8dab2942fb36abbf9e: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-3b3082d181004353bdff4414cd264d7c received from {username='slave'} at 127.27.220.138:55391
I20240502 14:17:34.590301 12461 tablet_copy_service.cc:433] P eddeff583cf34c8dab2942fb36abbf9e: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-3b3082d181004353bdff4414cd264d7c on tablet 3b3082d181004353bdff4414cd264d7c with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:34.624624 14885 raft_consensus.cc:1203] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:34.713006 15295 ts_tablet_manager.cc:902] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:34.714439 15295 tablet_copy_client.cc:308] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:34491
I20240502 14:17:34.715126 12461 tablet_copy_service.cc:140] P eddeff583cf34c8dab2942fb36abbf9e: Received BeginTabletCopySession request for tablet c9ff45109a6d4dc9aba4617819929981 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:55391)
I20240502 14:17:34.715271 12461 tablet_copy_service.cc:161] P eddeff583cf34c8dab2942fb36abbf9e: Beginning new tablet copy session on tablet c9ff45109a6d4dc9aba4617819929981 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:55391: session id = 1ba6a238e6ad4e4a84eff746ff336144-c9ff45109a6d4dc9aba4617819929981
I20240502 14:17:34.717157 12461 tablet_copy_source_session.cc:205] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:34.718230 15295 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:34.722681 15295 tablet_copy_client.cc:783] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:34.722925 15295 tablet_copy_client.cc:647] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:34.724303 15295 tablet_copy_client.cc:523] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:34.726840 15295 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:34.743402 14885 raft_consensus.cc:1203] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:34.762251 15295 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:34.762753 15295 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:34.763016 15295 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.036s user 0.034s sys 0.000s
I20240502 14:17:34.763672 15295 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.763904 15295 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:34.764048 15295 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:34.764616 15295 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 14:17:34.765467 12461 tablet_copy_service.cc:342] P eddeff583cf34c8dab2942fb36abbf9e: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-c9ff45109a6d4dc9aba4617819929981 received from {username='slave'} at 127.27.220.138:55391
I20240502 14:17:34.765626 12461 tablet_copy_service.cc:433] P eddeff583cf34c8dab2942fb36abbf9e: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-c9ff45109a6d4dc9aba4617819929981 on tablet c9ff45109a6d4dc9aba4617819929981 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:34.914250 15312 raft_consensus.cc:1050] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:34.915572 15312 consensus_queue.cc:237] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:34.918836 12572 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:34.918959 14885 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:34.919106 12440 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:34.920033 15310 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:34.920421 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:34.924984 15310 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:34.930639 12306 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:34.930742 13096 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:34.931280 15310 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:34.931874 15310 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:34.933683 12964 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:34.934353 15438 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:34.934763 12702 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:34.938666 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:34.944384 14328 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.948477 12572 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.948554 13096 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.950676 12440 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.951982 12964 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.953176 12166 catalog_manager.cc:5484] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:34.954353 14885 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.957417 12306 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.972301 12702 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.981911 12832 consensus_queue.cc:237] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:34.987349 12306 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:34.987411 12572 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:34.987437 13096 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:34.988102 14328 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:34.988435 15438 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:34.990024 12964 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:34.990583 15438 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:34.993085 15450 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.993614 12964 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.994112 12440 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:34.994686 14885 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:34.995476 15310 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:34.996021 15310 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:34.996320 15438 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:34.997462 12155 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b33c5e313e784b97987eb07b04f615f8 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:34.997838 12309 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:34.998502 12572 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.002035 13096 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.003214 14885 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.003334 12440 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.009092 12161 catalog_manager.cc:5484] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.016583 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet b33c5e313e784b97987eb07b04f615f8 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:35.017563 15451 tablet_replica.cc:330] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:35.018229 15451 raft_consensus.cc:2229] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:35.018745 15451 raft_consensus.cc:2258] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:35.020824 15451 ts_tablet_manager.cc:1875] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:35.023914 14885 raft_consensus.cc:1203] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:35.028029 15451 ts_tablet_manager.cc:1888] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:35.028184 15451 log.cc:1196] T b33c5e313e784b97987eb07b04f615f8 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/b33c5e313e784b97987eb07b04f615f8
I20240502 14:17:35.029006 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet b33c5e313e784b97987eb07b04f615f8 (table rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]) successfully deleted
I20240502 14:17:35.045936 15249 raft_consensus.cc:1050] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:35.047094 15249 consensus_queue.cc:237] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.049885 12309 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.050041 14885 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.050189 13096 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.050261 12440 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.050588 15236 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.050855 15453 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.051263 14659 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.051887 15455 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.052641 12832 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.053345 15453 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.060828 12702 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.062077 12964 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.063283 15249 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.063671 15454 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.064831 15249 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.067178 12964 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.067392 12702 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.067360 14885 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.070905 12309 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.071916 12160 catalog_manager.cc:5484] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:35.072394 12832 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.074674 13096 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.085012 12440 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.088037 12572 consensus_queue.cc:237] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.090655 14885 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.091117 13096 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.091122 12832 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.091341 15455 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.091630 12702 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.091696 15453 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.092151 15249 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.092373 15453 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
I20240502 14:17:35.093297 12309 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.094166 15455 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.096822 12440 raft_consensus.cc:1261] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.097502 15455 consensus_queue.cc:1035] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.097229 15454 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.097734 12702 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.097672 12306 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.101761 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 5614c0ba0e374af1853bd2fa586d3a76 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:35.102510 12832 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.103526 14885 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.106433 13096 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.107868 12165 catalog_manager.cc:5484] T 5614c0ba0e374af1853bd2fa586d3a76 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.101069 12440 raft_consensus.cc:2932] T 5614c0ba0e374af1853bd2fa586d3a76 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.115370 12944 tablet_service.cc:1514] Processing DeleteTablet for tablet 5614c0ba0e374af1853bd2fa586d3a76 with delete_type TABLET_DATA_TOMBSTONED (TS d693e089cfa54fbb8df91939410b19db not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:34406
I20240502 14:17:35.116384 15464 tablet_replica.cc:330] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: stopping tablet replica
I20240502 14:17:35.117089 15464 raft_consensus.cc:2229] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:35.117784 15464 raft_consensus.cc:2258] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:35.119917 15464 ts_tablet_manager.cc:1875] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:35.128433 15464 ts_tablet_manager.cc:1888] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:35.128614 15464 log.cc:1196] T 5614c0ba0e374af1853bd2fa586d3a76 P d693e089cfa54fbb8df91939410b19db: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/wals/5614c0ba0e374af1853bd2fa586d3a76
I20240502 14:17:35.129715 12153 catalog_manager.cc:4830] TS d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): tablet 5614c0ba0e374af1853bd2fa586d3a76 (table rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]) successfully deleted
I20240502 14:17:35.130363 14659 raft_consensus.cc:1050] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:35.131510 14659 consensus_queue.cc:237] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.134918 14885 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.136448 12832 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.136715 12440 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.137089 12701 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.136004 15454 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.002s
I20240502 14:17:35.137841 14659 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.138118 12964 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.138196 13096 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.138166 15455 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.138434 15236 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.138935 15454 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.139247 15236 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.141790 12306 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.142385 15454 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.169054 12832 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.167973 15454 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.175606 14884 raft_consensus.cc:1203] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:35.176563 12440 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.176740 12701 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.183997 12167 catalog_manager.cc:5484] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.187917 12962 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.195420 12306 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.200873 15267 raft_consensus.cc:1050] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:35.202284 15267 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.203871 14885 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.207077 14884 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.207651 15261 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.207609 12962 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.208891 13096 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.209720 15261 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.213339 12701 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.214073 15485 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.214972 12832 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.215943 12440 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.217162 15261 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.217742 12571 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.217826 15261 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.219599 12571 consensus_queue.cc:237] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.224232 12440 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.225044 15249 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.227329 12834 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.227509 14885 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.227965 15236 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.228436 15236 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.229208 12702 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.229741 12309 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.231146 15453 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.235574 13093 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.237368 15236 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.239959 15236 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.249572 15453 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.252310 12702 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.252067 12834 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.261634 15486 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.262207 12167 catalog_manager.cc:5484] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.262115 13093 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.264204 12306 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.266132 15267 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.268888 15267 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.269336 12944 tablet_service.cc:1514] Processing DeleteTablet for tablet ddf70e508caa4f48a9255be579060d97 with delete_type TABLET_DATA_TOMBSTONED (TS d693e089cfa54fbb8df91939410b19db not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:34406
I20240502 14:17:35.269933 15464 tablet_replica.cc:330] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: stopping tablet replica
I20240502 14:17:35.270128 15464 raft_consensus.cc:2229] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:35.270463 15464 raft_consensus.cc:2258] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:35.272300 12306 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.274215 15464 ts_tablet_manager.cc:1875] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:35.273990 12309 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.277838 12834 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.278388 12701 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.281217 15464 ts_tablet_manager.cc:1888] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:35.282732 15464 log.cc:1196] T ddf70e508caa4f48a9255be579060d97 P d693e089cfa54fbb8df91939410b19db: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/wals/ddf70e508caa4f48a9255be579060d97
I20240502 14:17:35.284796 12153 catalog_manager.cc:4830] TS d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): tablet ddf70e508caa4f48a9255be579060d97 (table rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]) successfully deleted
I20240502 14:17:35.286614 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ddf70e508caa4f48a9255be579060d97 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:35.287292 12164 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.288422 14884 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.291647 12440 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.296309 12571 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
W20240502 14:17:35.326511 13030 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.130:34491, user_credentials={real_user=slave}} blocked reactor thread for 110029us
I20240502 14:17:35.333081 12440 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.333954 14884 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
W20240502 14:17:35.343254 12901 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:45069, user_credentials={real_user=slave}} blocked reactor thread for 45586.5us
I20240502 14:17:35.345860 12964 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.601055 12834 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:35.604069 12571 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.604123 12702 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.604751 15450 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.604802 12964 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.605058 15438 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.605708 15438 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.605897 12440 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.606609 14328 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.607683 13093 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.608386 15438 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.610507 12306 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.616564 15438 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.624294 15272 raft_consensus.cc:1050] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:35.626148 15272 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.630620 14884 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.631947 12572 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.631022 15450 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.631691 12702 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.632969 12309 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.635876 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.638209 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.637744 12570 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.642166 12964 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.642302 12440 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.645244 12167 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.654589 12306 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.656147 15286 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.657246 12834 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.658332 12962 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.668824 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.669807 15271 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.669729 12834 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:35.674422 12570 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.674460 12964 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.675072 15450 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.675437 15438 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.671115 13093 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.693289 12306 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.693720 13096 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.697103 15536 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.697649 15525 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.700712 12702 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.702610 15271 raft_consensus.cc:1050] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:35.702124 15546 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:35.703630 15539 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.704267 15271 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.706193 12306 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:35.706410 12962 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.707861 15537 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.708926 13093 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:35.709697 14884 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.710482 12702 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:35.711725 12832 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.711611 15554 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.712114 12964 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.712906 15272 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.713413 15272 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.714908 12165 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.717792 12162 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:35.725237 12155 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 4ae8e0c47ce54724899b56aeb1b31336 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:35.727176 12570 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.727041 12965 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:35.727923 12570 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:35.729269 13096 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.733441 12570 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.733575 15555 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.734000 15485 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.738237 14884 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.742035 14885 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.742655 12439 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.742076 12833 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.743531 15491 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.744423 15491 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.748159 12834 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.749688 12570 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.754398 12306 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.755097 15491 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.756258 15261 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.757591 15261 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.760490 12309 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.762521 14883 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
W20240502 14:17:35.765614 12376 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.129:46613, user_credentials={real_user=slave}} blocked reactor thread for 110244us
I20240502 14:17:35.766312 13096 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.767520 12702 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.768488 13096 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.769596 12702 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.770826 12308 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:35.773204 12439 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.773383 12164 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
W20240502 14:17:35.776340 12769 consensus_peers.cc:485] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4ae8e0c47ce54724899b56aeb1b31336. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:35.777091 15546 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.778857 15286 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.779944 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet df48755615874edab65577e92249fc78 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:35.780052 15286 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.781105 15286 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.782191 15286 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.783207 15286 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:35.788715 12962 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.789316 15491 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.789681 12833 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.791890 12964 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.795560 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet df48755615874edab65577e92249fc78 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:35.796847 12440 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.796255 12570 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.806537 12702 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.823014 12307 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.833925 15562 tablet_replica.cc:330] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:35.834275 15562 raft_consensus.cc:2229] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:35.834592 15562 raft_consensus.cc:2258] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:35.834030 12440 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:35.835914 15562 ts_tablet_manager.cc:1875] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:35.849323 15555 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.849699 12832 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.849602 12569 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.851274 13093 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.857372 12701 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.860692 12962 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.862778 14885 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.865666 15562 ts_tablet_manager.cc:1888] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:35.865878 15562 log.cc:1196] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/df48755615874edab65577e92249fc78
I20240502 14:17:35.867597 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet df48755615874edab65577e92249fc78 (table rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]) successfully deleted
I20240502 14:17:35.868870 12159 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.872227 12307 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.872479 13096 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.879467 12570 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:35.885218 12832 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.885804 12702 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.885967 15455 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.886396 13093 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.886788 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.887064 15455 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.889264 12307 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.889896 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.898486 12962 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.899114 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.902236 12440 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:35.917982 12440 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:35.919095 14659 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.921447 12833 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.926288 12306 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.927979 12161 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.930145 13096 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.932880 12701 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.932425 12962 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.937361 14885 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.937785 15454 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:35.938845 15558 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.945456 12570 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:35.951745 12964 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.952519 15558 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.953723 12569 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.954357 12833 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.956889 15539 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.957479 15539 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.957167 12439 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:35.959455 13093 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.960925 15286 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.962060 12306 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:35.962760 15555 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.963310 12569 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.964959 15286 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:35.967015 13094 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.967052 12833 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.971524 12306 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.975759 12166 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:35.979784 12832 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.980645 12702 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.981334 15249 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.981781 12307 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.982477 15249 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.984097 15454 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:35.985116 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3b3082d181004353bdff4414cd264d7c with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:35.988092 12964 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:35.992940 15249 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.003692 12439 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.003037 14885 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.006726 12306 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.007423 12964 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.010589 12702 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.019414 12160 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.020298 12832 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.022656 13093 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.027017 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.030045 13093 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.032958 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet 3b3082d181004353bdff4414cd264d7c with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:36.033346 15562 tablet_replica.cc:330] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:36.033599 15562 raft_consensus.cc:2229] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:36.033854 15562 raft_consensus.cc:2258] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:36.035097 15562 ts_tablet_manager.cc:1875] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:36.042713 15562 ts_tablet_manager.cc:1888] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:36.042905 15562 log.cc:1196] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/3b3082d181004353bdff4414cd264d7c
I20240502 14:17:36.044031 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet 3b3082d181004353bdff4414cd264d7c (table rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]) successfully deleted
I20240502 14:17:36.046660 12152 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet cd382683ec2b4600b6edaced79698b7e with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:36.047186 15454 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.047567 15249 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.050410 12439 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.051638 12964 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
W20240502 14:17:36.065513 12506 consensus_peers.cc:485] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd382683ec2b4600b6edaced79698b7e. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:36.075785 13093 consensus_queue.cc:237] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:36.078488 12306 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.078508 12570 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.078966 12833 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.079124 12702 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.079053 15267 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.079696 15267 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.080060 15261 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.080437 12439 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.080770 12964 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.081039 15261 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.081355 15267 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.084635 15267 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.100929 15261 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.108772 12702 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.109476 12306 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.110625 12964 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.111173 12159 catalog_manager.cc:5484] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:36.114914 12439 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.123829 13093 consensus_queue.cc:237] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:36.127498 12833 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.130038 12964 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.131481 12440 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.132318 15495 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.133193 15267 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.135118 12569 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.136018 15261 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.137856 12569 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.148674 12570 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.152128 12832 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.157151 15267 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.157969 12702 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.159143 12306 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.159616 15491 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.160962 15267 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.162572 12702 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.162781 12161 catalog_manager.cc:5484] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.166345 15261 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 14:17:36.166967 13031 consensus_peers.cc:485] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 01ffac38da1d4ece9cba649a59d5518b. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:36.167047 12154 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 01ffac38da1d4ece9cba649a59d5518b with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:36.159205 12833 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.168427 12306 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.174419 12439 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.174947 12964 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.250416 15625 ts_tablet_manager.cc:902] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069)
I20240502 14:17:36.252764 15625 tablet_copy_client.cc:308] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:45069
I20240502 14:17:36.258908 12854 tablet_copy_service.cc:140] P c78bd31d8d2142bf9d8e06b5a0074e0f: Received BeginTabletCopySession request for tablet 4ae8e0c47ce54724899b56aeb1b31336 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:34427)
I20240502 14:17:36.259088 12854 tablet_copy_service.cc:161] P c78bd31d8d2142bf9d8e06b5a0074e0f: Beginning new tablet copy session on tablet 4ae8e0c47ce54724899b56aeb1b31336 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:34427: session id = 4c04a4a8d8864267be7f67c2abe98f84-4ae8e0c47ce54724899b56aeb1b31336
I20240502 14:17:36.261669 12854 tablet_copy_source_session.cc:205] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:36.266294 15625 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:36.277413 15625 tablet_copy_client.cc:783] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:36.277938 15625 tablet_copy_client.cc:647] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:36.279867 15625 tablet_copy_client.cc:523] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:36.283106 15625 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:36.292477 15625 log.cc:824] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:36.332288 15625 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:36.332787 15625 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:36.333194 15625 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.050s user 0.042s sys 0.004s
I20240502 14:17:36.338277 15625 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:36.338680 15625 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:36.339143 15625 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:36.340670 15625 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.007s user 0.006s sys 0.004s
I20240502 14:17:36.341589 12854 tablet_copy_service.cc:342] P c78bd31d8d2142bf9d8e06b5a0074e0f: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-4ae8e0c47ce54724899b56aeb1b31336 received from {username='slave'} at 127.27.220.136:34427
I20240502 14:17:36.341725 12854 tablet_copy_service.cc:433] P c78bd31d8d2142bf9d8e06b5a0074e0f: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-4ae8e0c47ce54724899b56aeb1b31336 on tablet 4ae8e0c47ce54724899b56aeb1b31336 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:36.367125 12439 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:36.369901 14885 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:36.370429 12832 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:36.371184 12570 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:36.371727 15539 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:36.372014 15555 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:36.372388 15558 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:36.374642 13093 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:36.374755 12306 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:36.374764 12702 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:36.375748 15539 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:36.376216 15539 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:36.377224 15555 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:36.380282 14885 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.380982 12570 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.384111 15271 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.386195 12166 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.389075 12702 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.389087 12306 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.393659 12944 tablet_service.cc:1514] Processing DeleteTablet for tablet c9ff45109a6d4dc9aba4617819929981 with delete_type TABLET_DATA_TOMBSTONED (TS d693e089cfa54fbb8df91939410b19db not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:34406
I20240502 14:17:36.395347 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c9ff45109a6d4dc9aba4617819929981 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:36.396332 12832 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.397634 15634 tablet_replica.cc:330] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: stopping tablet replica
I20240502 14:17:36.397912 15634 raft_consensus.cc:2229] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:36.398262 15634 raft_consensus.cc:2258] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:36.399286 15634 ts_tablet_manager.cc:1875] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:36.401163 13093 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:36.406528 15634 ts_tablet_manager.cc:1888] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:36.406747 15634 log.cc:1196] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/wals/c9ff45109a6d4dc9aba4617819929981
I20240502 14:17:36.409183 12153 catalog_manager.cc:4830] TS d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): tablet c9ff45109a6d4dc9aba4617819929981 (table rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]) successfully deleted
I20240502 14:17:36.458923 15641 ts_tablet_manager.cc:902] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:36.463668 15641 tablet_copy_client.cc:308] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:36.466681 12964 consensus_queue.cc:237] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:36.470233 12570 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.471244 12439 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.471560 13093 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.471875 12832 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.471952 12306 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.473248 12702 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.473485 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet cd382683ec2b4600b6edaced79698b7e from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:45555)
I20240502 14:17:36.473757 15501 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.473906 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet cd382683ec2b4600b6edaced79698b7e from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:45555: session id = 95e58ca33b4048e2b000dfbf2b5a0186-cd382683ec2b4600b6edaced79698b7e
I20240502 14:17:36.474800 15501 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.475561 15631 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.477269 12592 tablet_copy_source_session.cc:205] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:36.484189 15641 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:36.485298 15650 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.488684 15652 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.001s
I20240502 14:17:36.492338 15650 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.500648 15631 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.507158 15641 tablet_copy_client.cc:783] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:36.506758 12570 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.507664 15641 tablet_copy_client.cc:647] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:36.508260 12439 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.508297 13093 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.511711 12160 catalog_manager.cc:5484] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:36.513736 15641 tablet_copy_client.cc:523] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:36.522387 15641 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
W20240502 14:17:36.531891 15650 consensus_queue.cc:1159] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "53479f0fb09c454e9b3cbe7398bf8874" responder_term: 1 status { last_received { term: 1 index: 4 } last_committed_idx: 4 last_received_current_leader { term: 1 index: 4 } }
W20240502 14:17:36.534847 12901 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.133:45069, user_credentials={real_user=slave}} blocked reactor thread for 62365.2us
I20240502 14:17:36.538412 12964 consensus_queue.cc:237] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:36.541574 15641 log.cc:824] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: Log is configured to *not* fsync() on all Append() calls
I20240502 14:17:36.543673 13093 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.543726 12306 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.544588 12702 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.544684 12570 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.545249 15650 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.545576 15631 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.545974 15631 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.546531 15631 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.547483 12306 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.548336 12702 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.553257 12832 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.553938 15631 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.557405 12832 raft_consensus.cc:858] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }.
Pending config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }:
New config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
W20240502 14:17:36.557556 12832 raft_consensus.cc:1565] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Could not prepare op 'term: 1 index: 5' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }.
Pending config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }:
New config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:36.559106 15658 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.559614 12832 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.567034 12153 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:36.572248 12833 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.578800 12309 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.579653 12570 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.579938 12702 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.582947 13094 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.588053 12161 catalog_manager.cc:5484] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.601610 15641 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:36.602262 15641 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:36.602569 12439 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.602907 15641 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.081s user 0.066s sys 0.000s
I20240502 14:17:36.607741 15631 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.610146 12439 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:36.610560 15641 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:36.611377 15641 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:36.611927 15641 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:36.613588 15641 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.011s user 0.008s sys 0.001s
I20240502 14:17:36.615134 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-cd382683ec2b4600b6edaced79698b7e received from {username='slave'} at 127.27.220.137:45555
I20240502 14:17:36.615286 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-cd382683ec2b4600b6edaced79698b7e on tablet cd382683ec2b4600b6edaced79698b7e with peer 95e58ca33b4048e2b000dfbf2b5a0186
W20240502 14:17:36.623564 12899 consensus_peers.cc:485] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0c4323e7ed9d4784ba7f8dd331036fb3. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:36.626060 12964 consensus_queue.cc:237] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:36.628669 12569 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.628846 12439 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.628875 12833 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.629135 15652 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.629244 12306 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.629823 15652 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.629855 13094 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.630300 15652 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.630396 12702 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:36.630929 15631 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.631191 15650 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.631680 15650 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:36.636953 15652 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.639834 12702 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.641253 12833 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.641491 12306 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.641903 13094 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.645737 15625 ts_tablet_manager.cc:902] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:36.647141 12165 catalog_manager.cc:5484] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:36.652726 12439 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.661932 12569 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:36.664202 12964 consensus_queue.cc:237] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:36.667374 15625 tablet_copy_client.cc:308] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:36.675463 13116 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet 01ffac38da1d4ece9cba649a59d5518b from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:52275)
I20240502 14:17:36.675678 13116 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet 01ffac38da1d4ece9cba649a59d5518b from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:52275: session id = 4c04a4a8d8864267be7f67c2abe98f84-01ffac38da1d4ece9cba649a59d5518b
I20240502 14:17:36.677491 12439 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.677587 12569 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.678151 15652 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.678781 13116 tablet_copy_source_session.cc:205] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:36.678736 15652 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.681321 15625 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:36.681346 12833 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.682132 15649 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.682958 12702 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.683614 13094 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.683934 15652 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.685927 15650 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.688290 12306 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:36.688514 14746 raft_consensus.cc:1203] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:36.688805 15652 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:36.688501 15625 tablet_copy_client.cc:783] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:36.690286 15625 tablet_copy_client.cc:647] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:36.691996 15625 tablet_copy_client.cc:523] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:36.691597 15649 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.694370 12440 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.695071 12702 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.697216 12153 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 29809afc38e54766ba33a4e24311f8f4 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:36.698449 12832 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.698798 12569 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.702230 12160 catalog_manager.cc:5484] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.704623 15625 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:36.710826 12306 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:36.715703 13094 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
W20240502 14:17:36.718489 12900 consensus_peers.cc:485] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 29809afc38e54766ba33a4e24311f8f4. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:36.776366 15625 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:36.776875 15625 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:36.777179 15625 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.073s user 0.030s sys 0.011s
I20240502 14:17:36.778136 15625 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:36.778450 15625 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:36.778645 15625 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:36.779211 15625 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:17:36.780138 13116 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-01ffac38da1d4ece9cba649a59d5518b received from {username='slave'} at 127.27.220.136:52275
I20240502 14:17:36.780274 13116 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-01ffac38da1d4ece9cba649a59d5518b on tablet 01ffac38da1d4ece9cba649a59d5518b with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:37.033984 15016 raft_consensus.cc:1203] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:37.065433 15641 ts_tablet_manager.cc:902] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:37.066663 15641 tablet_copy_client.cc:308] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:37.072844 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:45165)
I20240502 14:17:37.073025 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet 0c4323e7ed9d4784ba7f8dd331036fb3 from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:45165: session id = 95e58ca33b4048e2b000dfbf2b5a0186-0c4323e7ed9d4784ba7f8dd331036fb3
I20240502 14:17:37.075621 12985 tablet_copy_source_session.cc:205] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:37.077248 15641 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c4323e7ed9d4784ba7f8dd331036fb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:37.082346 15641 tablet_copy_client.cc:783] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:37.082978 15641 tablet_copy_client.cc:647] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:37.084908 15641 tablet_copy_client.cc:523] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:37.087639 15641 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:37.111348 15625 ts_tablet_manager.cc:902] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:37.113094 15625 tablet_copy_client.cc:308] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:37.120777 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet 29809afc38e54766ba33a4e24311f8f4 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:58685)
I20240502 14:17:37.121014 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet 29809afc38e54766ba33a4e24311f8f4 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:58685: session id = 4c04a4a8d8864267be7f67c2abe98f84-29809afc38e54766ba33a4e24311f8f4
I20240502 14:17:37.123673 12985 tablet_copy_source_session.cc:205] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:37.124967 15625 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:37.133172 15625 tablet_copy_client.cc:783] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:37.133538 15625 tablet_copy_client.cc:647] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:37.135383 15641 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:37.135834 15641 tablet_bootstrap.cc:492] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:37.135804 15625 tablet_copy_client.cc:523] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:37.136121 15641 ts_tablet_manager.cc:1367] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.049s user 0.045s sys 0.000s
I20240502 14:17:37.137020 15641 raft_consensus.cc:348] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:37.137269 15641 raft_consensus.cc:729] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:37.137493 15641 consensus_queue.cc:260] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:37.138605 15641 ts_tablet_manager.cc:1398] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.001s sys 0.004s
I20240502 14:17:37.138826 15625 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:37.139492 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-0c4323e7ed9d4784ba7f8dd331036fb3 received from {username='slave'} at 127.27.220.137:45165
I20240502 14:17:37.139684 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-0c4323e7ed9d4784ba7f8dd331036fb3 on tablet 0c4323e7ed9d4784ba7f8dd331036fb3 with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:37.173911 15625 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:37.174337 15625 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:37.174612 15625 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.036s user 0.029s sys 0.004s
I20240502 14:17:37.175271 15625 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:37.175515 15625 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:37.175740 15625 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:37.176301 15625 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:37.177608 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-29809afc38e54766ba33a4e24311f8f4 received from {username='slave'} at 127.27.220.136:58685
I20240502 14:17:37.177714 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-29809afc38e54766ba33a4e24311f8f4 on tablet 29809afc38e54766ba33a4e24311f8f4 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:37.222002 15684 raft_consensus.cc:1050] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:37.222883 15684 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:37.225946 13094 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.226001 14746 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.226733 12702 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.226994 12964 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.227186 12569 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.227967 12306 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.228631 12440 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.229239 15546 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.229841 15546 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.230314 15546 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.230695 15684 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.231427 15753 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.234265 15684 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.249821 15752 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.252614 14746 raft_consensus.cc:1203] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:37.253681 15546 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.259594 14746 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.261853 12440 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.261751 12306 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.268774 12160 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:37.276862 12702 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.280392 13094 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.283900 12569 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.285384 12964 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.353364 12833 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:37.356745 13094 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.356778 14746 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.357717 15751 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.358031 15754 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.358263 12440 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.358449 12569 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.358888 15754 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.359253 12306 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.359242 15752 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.361070 12702 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.361732 15751 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.363444 15684 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.363770 15546 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.366173 12702 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.367317 12306 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.370666 14746 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.365089 13094 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.377576 12166 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.383986 12155 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4ae8e0c47ce54724899b56aeb1b31336 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:37.386294 12569 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.390640 12440 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.396499 12944 tablet_service.cc:1514] Processing DeleteTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 with delete_type TABLET_DATA_TOMBSTONED (TS d693e089cfa54fbb8df91939410b19db not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:34406
I20240502 14:17:37.414259 15773 tablet_replica.cc:330] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: stopping tablet replica
I20240502 14:17:37.414623 15773 raft_consensus.cc:2229] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:37.415611 15773 raft_consensus.cc:2258] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:37.418574 15773 ts_tablet_manager.cc:1875] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:37.426151 15773 ts_tablet_manager.cc:1888] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:37.426311 15773 log.cc:1196] T 4ae8e0c47ce54724899b56aeb1b31336 P d693e089cfa54fbb8df91939410b19db: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/wals/4ae8e0c47ce54724899b56aeb1b31336
I20240502 14:17:37.427126 12153 catalog_manager.cc:4830] TS d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): tablet 4ae8e0c47ce54724899b56aeb1b31336 (table rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]) successfully deleted
I20240502 14:17:37.444322 15454 raft_consensus.cc:1050] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:37.445586 15454 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:37.448983 15016 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.449002 13094 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.449648 12833 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.449877 15249 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.450282 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.450872 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.460585 12702 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.460624 12309 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.466164 15788 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.466890 15788 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.466975 12964 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.467610 12440 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.468259 15249 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.469167 15249 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.501179 15788 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.501958 12962 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.502995 12306 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.508296 15016 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.509335 12832 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.515091 12167 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 reported cstate change: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.518592 15015 raft_consensus.cc:1203] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:37.518330 12702 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.522878 12440 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.522984 13094 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.536211 12569 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:37.541179 12309 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.541199 12832 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.541819 15788 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.541946 13094 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.543135 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.543224 12440 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.543759 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.544385 14659 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.547749 15016 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.547780 15788 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.548462 12439 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.552242 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd382683ec2b4600b6edaced79698b7e with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:37.553620 12832 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.553911 12306 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.555043 12162 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:37.559069 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet cd382683ec2b4600b6edaced79698b7e with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:37.560003 15801 tablet_replica.cc:330] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:37.560250 15801 raft_consensus.cc:2229] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:37.560555 15801 raft_consensus.cc:2258] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:37.561643 15801 ts_tablet_manager.cc:1875] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:37.553248 13094 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.563623 15249 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.567004 15016 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.568370 15801 ts_tablet_manager.cc:1888] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:37.568509 15801 log.cc:1196] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/cd382683ec2b4600b6edaced79698b7e
I20240502 14:17:37.569393 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet cd382683ec2b4600b6edaced79698b7e (table rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]) successfully deleted
I20240502 14:17:37.576109 12964 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.576737 15249 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.578389 12964 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:37.601053 14746 raft_consensus.cc:1203] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:37.679298 15491 raft_consensus.cc:1050] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:37.680207 15491 consensus_queue.cc:237] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:37.683792 12569 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.685410 12832 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.686096 12702 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.686091 15803 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.686673 15803 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.687281 12439 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.687686 12306 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.688668 14746 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.688846 15491 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.689193 12962 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.689335 15597 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.690001 15597 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.690312 15804 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.692369 15491 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.711649 15491 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.714808 12440 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.720849 12569 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.721204 14746 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.722431 12306 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.728425 12167 catalog_manager.cc:5484] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 reported cstate change: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.728569 12702 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.732733 12832 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.739885 12962 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.754845 13094 consensus_queue.cc:237] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:37.758448 14746 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.759148 15805 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.759452 12306 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.760309 12569 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.761072 12832 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.762024 15491 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.762652 15491 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.764076 12440 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.764114 15803 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.765461 15491 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.768440 15805 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.770059 12569 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.770813 12700 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:37.770721 12306 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.771423 15804 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:37.773051 12700 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.778450 12166 catalog_manager.cc:5484] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.777942 12832 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.781863 14746 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.783491 12440 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:37.779619 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 01ffac38da1d4ece9cba649a59d5518b with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:37.792941 12944 tablet_service.cc:1514] Processing DeleteTablet for tablet 01ffac38da1d4ece9cba649a59d5518b with delete_type TABLET_DATA_TOMBSTONED (TS d693e089cfa54fbb8df91939410b19db not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:34406
I20240502 14:17:37.793495 15773 tablet_replica.cc:330] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: stopping tablet replica
I20240502 14:17:37.793737 15773 raft_consensus.cc:2229] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:37.794013 15773 raft_consensus.cc:2258] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:37.795135 15773 ts_tablet_manager.cc:1875] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:37.801584 15773 ts_tablet_manager.cc:1888] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:37.801813 15773 log.cc:1196] T 01ffac38da1d4ece9cba649a59d5518b P d693e089cfa54fbb8df91939410b19db: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-5/wal/wals/01ffac38da1d4ece9cba649a59d5518b
I20240502 14:17:37.802774 12153 catalog_manager.cc:4830] TS d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): tablet 01ffac38da1d4ece9cba649a59d5518b (table rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]) successfully deleted
I20240502 14:17:37.960139 15501 raft_consensus.cc:1050] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:37.961110 15501 consensus_queue.cc:237] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:37.963980 12440 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.964195 12569 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.964464 12306 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.964788 15837 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.965402 15838 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.965914 15016 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.965927 15839 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.968001 13094 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.967593 15840 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
I20240502 14:17:37.970559 12832 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.971115 12700 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:37.971375 15837 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.971791 15839 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:37.972086 15652 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:38.000757 15839 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.002529 12569 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.002506 12833 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.007200 12440 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.010668 12165 catalog_manager.cc:5484] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:38.011986 12700 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.013964 12306 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.020290 13093 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.036192 15016 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.040490 12962 consensus_queue.cc:237] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:38.045182 12440 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.046034 15837 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.046536 12569 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.047780 15838 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.047842 15016 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.049587 12306 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.049829 13093 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.052166 15652 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.053807 12832 raft_consensus.cc:1261] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.054364 15839 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.055682 15652 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.056035 15839 consensus_queue.cc:1035] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.057173 15838 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.059305 13093 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.059321 15016 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.061090 12832 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.061491 12306 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.066570 12164 catalog_manager.cc:5484] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 95e58ca33b4048e2b000dfbf2b5a0186 reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.068958 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:38.070609 12440 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.074235 12569 raft_consensus.cc:2932] T 0c4323e7ed9d4784ba7f8dd331036fb3 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:38.077572 12306 consensus_queue.cc:237] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:38.078011 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet 0c4323e7ed9d4784ba7f8dd331036fb3 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:38.085466 13093 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.086454 12569 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.086555 12962 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.087236 15813 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.087760 12832 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.087886 15813 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.088095 12700 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.088230 12440 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.089160 15858 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.089920 15748 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.090188 15858 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.108933 15858 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:38.111424 12832 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:38.114732 12962 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:38.118510 15748 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.122480 12165 catalog_manager.cc:5484] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:38.121939 12440 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:38.123565 13093 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:38.131759 12306 consensus_queue.cc:237] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:38.134936 15855 tablet_replica.cc:330] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:38.137959 15855 raft_consensus.cc:2229] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:38.138548 12962 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.138666 12440 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.138543 15855 raft_consensus.cc:2258] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:38.139530 15857 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.139797 15858 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.140704 15855 ts_tablet_manager.cc:1875] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:38.143697 12700 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:38.145864 12569 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.146064 12832 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.146648 15813 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.149492 12570 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:38.150166 15855 ts_tablet_manager.cc:1888] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:38.150416 15855 log.cc:1196] T 0c4323e7ed9d4784ba7f8dd331036fb3 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/0c4323e7ed9d4784ba7f8dd331036fb3
I20240502 14:17:38.151237 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet 0c4323e7ed9d4784ba7f8dd331036fb3 (table rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]) successfully deleted
I20240502 14:17:38.152436 15858 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.152912 13093 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.153767 15857 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.155828 15748 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.157733 12701 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.158246 12833 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.159797 13093 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.160300 12569 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.161648 15813 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.164968 12154 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 0397d318dc56416c9d8043f98462aeee with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:38.166347 12167 catalog_manager.cc:5484] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.171186 12440 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.174495 12962 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
W20240502 14:17:38.175415 12243 consensus_peers.cc:485] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0397d318dc56416c9d8043f98462aeee. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:38.173437 12700 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.192672 15840 raft_consensus.cc:1050] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:38.193997 15840 consensus_queue.cc:237] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:38.197393 14746 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:38.198783 15838 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:38.199191 12440 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:38.199260 12306 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:38.199752 12701 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:38.199753 15501 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:38.200578 12833 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:38.200698 12570 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:38.201426 15501 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:38.201876 15837 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:38.202435 13093 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:38.208469 15840 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:38.208711 15837 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:38.209192 15837 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:38.240764 15837 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.241508 14746 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.245576 12570 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.247196 12164 catalog_manager.cc:5484] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 reported cstate change: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.254396 12306 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.261603 12439 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.263186 13093 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.264274 12702 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.272192 12832 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.277549 12962 consensus_queue.cc:237] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:38.280745 12570 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.280819 12832 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.280891 14746 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.281392 15841 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.281915 15840 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.282789 12306 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.284799 15649 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.285116 12439 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.285471 15649 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.285979 13093 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:38.286055 15649 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.286617 15649 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:38.287849 15649 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.289450 12306 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.291006 14746 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.292933 12165 catalog_manager.cc:5484] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:38.294359 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 29809afc38e54766ba33a4e24311f8f4 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:38.295738 12570 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.298235 12832 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.299412 12440 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.307505 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet 29809afc38e54766ba33a4e24311f8f4 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:38.307931 15855 tablet_replica.cc:330] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:38.308145 15855 raft_consensus.cc:2229] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:38.308425 15855 raft_consensus.cc:2258] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:38.310287 15855 ts_tablet_manager.cc:1875] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:38.311009 13093 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:38.316538 15855 ts_tablet_manager.cc:1888] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:38.316730 15855 log.cc:1196] T 29809afc38e54766ba33a4e24311f8f4 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/29809afc38e54766ba33a4e24311f8f4
I20240502 14:17:38.317531 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet 29809afc38e54766ba33a4e24311f8f4 (table rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]) successfully deleted
I20240502 14:17:38.506026 13093 consensus_queue.cc:237] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:38.508863 12306 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.508977 12570 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.509490 12832 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.509747 15491 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.510139 15820 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.510782 15820 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.515339 12439 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.515377 12702 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.515506 12962 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.515321 15901 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.516214 15820 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.516567 15902 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.517724 12832 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.519402 12702 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.521931 12306 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.522395 12570 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.523300 12164 catalog_manager.cc:5484] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:38.534343 15897 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.541667 12962 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.545580 13093 consensus_queue.cc:237] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:38.549338 12832 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.550257 12702 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.550508 12962 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.551179 15897 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.551442 15820 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.550803 12439 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.553447 15820 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.557057 12306 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.558418 15902 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.559832 12440 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.561674 15908 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.561924 12570 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.562063 15820 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
W20240502 14:17:38.562932 13030 consensus_peers.cc:485] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f8526552032e4a45ab4be4743f12abd0. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:38.563994 12440 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.565776 15897 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.566514 12306 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.567345 12570 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.569700 12154 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet f8526552032e4a45ab4be4743f12abd0 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:38.569483 12962 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.571914 12164 catalog_manager.cc:5484] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.577437 12832 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.589941 15910 ts_tablet_manager.cc:902] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:38.602239 12702 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:38.633711 15910 tablet_copy_client.cc:308] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:46613
I20240502 14:17:38.644266 12329 tablet_copy_service.cc:140] P 70a425a77fe54cbcb9755f9c540fd367: Received BeginTabletCopySession request for tablet 0397d318dc56416c9d8043f98462aeee from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:51903)
I20240502 14:17:38.644577 12329 tablet_copy_service.cc:161] P 70a425a77fe54cbcb9755f9c540fd367: Beginning new tablet copy session on tablet 0397d318dc56416c9d8043f98462aeee from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:51903: session id = 95e58ca33b4048e2b000dfbf2b5a0186-0397d318dc56416c9d8043f98462aeee
I20240502 14:17:38.648566 12329 tablet_copy_source_session.cc:205] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:38.650915 15910 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0397d318dc56416c9d8043f98462aeee. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:38.656780 15910 tablet_copy_client.cc:783] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:38.657168 15910 tablet_copy_client.cc:647] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:38.659250 15910 tablet_copy_client.cc:523] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:38.662420 15910 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:38.697935 15910 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:38.698421 15910 tablet_bootstrap.cc:492] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:38.698794 15910 ts_tablet_manager.cc:1367] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.036s user 0.034s sys 0.003s
I20240502 14:17:38.699642 15910 raft_consensus.cc:348] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:38.699961 15910 raft_consensus.cc:729] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:38.700167 15910 consensus_queue.cc:260] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:38.700985 15910 ts_tablet_manager.cc:1398] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:17:38.702368 12329 tablet_copy_service.cc:342] P 70a425a77fe54cbcb9755f9c540fd367: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-0397d318dc56416c9d8043f98462aeee received from {username='slave'} at 127.27.220.137:51903
I20240502 14:17:38.702536 12329 tablet_copy_service.cc:433] P 70a425a77fe54cbcb9755f9c540fd367: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-0397d318dc56416c9d8043f98462aeee on tablet 0397d318dc56416c9d8043f98462aeee with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:38.874749 12962 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:38.884075 12570 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.884711 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.885958 12306 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.886948 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.897879 13093 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.898614 12439 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.898862 12832 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.898850 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.899214 12702 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:38.900107 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.900406 15939 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.901284 15939 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:38.901979 15841 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.907477 13093 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.908919 12833 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.909703 12702 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.910997 12570 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.917655 12164 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.935647 12309 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.936084 12439 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:38.947770 12962 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
W20240502 14:17:38.952716 12900 consensus_peers.cc:485] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9cd04934b5264d7a9643974c0386cac4. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:38.953840 12833 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.953884 13093 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.954195 12439 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.954820 15939 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.955070 12570 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.955183 15950 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.955658 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.955922 15942 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.956868 12702 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.958348 15950 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.960026 12309 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:38.960547 15841 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:38.961403 15939 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:38.961319 12832 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:38.963493 12306 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:38.965447 12153 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 9cd04934b5264d7a9643974c0386cac4 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:38.967111 12570 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:38.968307 12702 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:38.968124 12164 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:38.972179 15910 ts_tablet_manager.cc:902] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:38.972443 13093 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:38.986640 15910 tablet_copy_client.cc:308] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:38.988670 12439 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:38.994783 13116 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet f8526552032e4a45ab4be4743f12abd0 from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:46251)
I20240502 14:17:38.995085 13116 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet f8526552032e4a45ab4be4743f12abd0 from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:46251: session id = 95e58ca33b4048e2b000dfbf2b5a0186-f8526552032e4a45ab4be4743f12abd0
I20240502 14:17:38.998178 13116 tablet_copy_source_session.cc:205] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:39.000973 15910 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f8526552032e4a45ab4be4743f12abd0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:39.006841 15910 tablet_copy_client.cc:783] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:39.007105 15910 tablet_copy_client.cc:647] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:39.009358 15910 tablet_copy_client.cc:523] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:39.011974 15910 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:39.022047 15016 raft_consensus.cc:1203] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:39.049875 15910 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:39.050304 15910 tablet_bootstrap.cc:492] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:39.050563 15910 ts_tablet_manager.cc:1367] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.039s user 0.030s sys 0.008s
I20240502 14:17:39.051285 15910 raft_consensus.cc:348] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:39.051546 15910 raft_consensus.cc:729] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:39.051759 15910 consensus_queue.cc:260] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:39.052322 15910 ts_tablet_manager.cc:1398] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:39.053309 13116 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-f8526552032e4a45ab4be4743f12abd0 received from {username='slave'} at 127.27.220.137:46251
I20240502 14:17:39.053483 13116 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-f8526552032e4a45ab4be4743f12abd0 on tablet f8526552032e4a45ab4be4743f12abd0 with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:39.348412 15016 raft_consensus.cc:1203] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:39.402365 15986 ts_tablet_manager.cc:902] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:39.403487 15986 tablet_copy_client.cc:308] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:39.404227 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet 9cd04934b5264d7a9643974c0386cac4 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:58685)
I20240502 14:17:39.404423 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet 9cd04934b5264d7a9643974c0386cac4 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:58685: session id = 4c04a4a8d8864267be7f67c2abe98f84-9cd04934b5264d7a9643974c0386cac4
I20240502 14:17:39.406654 12985 tablet_copy_source_session.cc:205] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:39.407789 15986 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:39.412199 15986 tablet_copy_client.cc:783] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:39.412441 15986 tablet_copy_client.cc:647] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:39.413877 15986 tablet_copy_client.cc:523] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:39.416417 15986 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:39.464088 15986 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:39.464672 15986 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:39.465010 15986 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.049s user 0.043s sys 0.004s
I20240502 14:17:39.466045 15986 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:39.466441 15986 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:39.466718 15986 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:39.482126 15986 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.017s user 0.007s sys 0.012s
I20240502 14:17:39.485190 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-9cd04934b5264d7a9643974c0386cac4 received from {username='slave'} at 127.27.220.136:58685
I20240502 14:17:39.485400 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-9cd04934b5264d7a9643974c0386cac4 on tablet 9cd04934b5264d7a9643974c0386cac4 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:39.594061 15858 raft_consensus.cc:1050] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:39.595023 15858 consensus_queue.cc:237] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:39.597999 13094 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.598631 16010 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.599448 15016 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.599500 12964 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.600229 16010 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.600416 12700 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.604197 16010 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.003s
I20240502 14:17:39.605054 16011 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.605645 12570 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.606971 12832 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.607558 16008 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.608093 16008 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.615967 16011 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.617343 12570 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.620671 12702 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.627429 12167 catalog_manager.cc:5484] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.628005 12439 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.628042 15016 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.628746 16010 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.630053 12832 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.634971 12439 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.635103 12964 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.647305 13094 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.657732 12309 consensus_queue.cc:237] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:39.661916 12964 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.661980 15016 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.662197 13094 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.662197 12570 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.662705 12439 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.662942 12833 raft_consensus.cc:1261] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.662882 16021 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.663514 16021 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.664256 16008 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.664526 16021 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.664912 15858 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.665163 16013 consensus_queue.cc:1035] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.668108 16008 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.668720 13094 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.669726 12833 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.672120 15016 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.674854 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0397d318dc56416c9d8043f98462aeee with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:39.676167 12570 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.676649 12165 catalog_manager.cc:5484] T 0397d318dc56416c9d8043f98462aeee P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.679463 12964 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.679749 12439 raft_consensus.cc:2932] T 0397d318dc56416c9d8043f98462aeee P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.687932 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet 0397d318dc56416c9d8043f98462aeee with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:39.690622 16026 tablet_replica.cc:330] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:39.690917 16026 raft_consensus.cc:2229] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:39.691196 16026 raft_consensus.cc:2258] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:39.691958 16026 ts_tablet_manager.cc:1875] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:39.697772 16026 ts_tablet_manager.cc:1888] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:39.697906 16026 log.cc:1196] T 0397d318dc56416c9d8043f98462aeee P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/0397d318dc56416c9d8043f98462aeee
I20240502 14:17:39.698639 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet 0397d318dc56416c9d8043f98462aeee (table rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]) successfully deleted
I20240502 14:17:39.753819 15969 raft_consensus.cc:1050] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:39.754689 15969 consensus_queue.cc:237] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:39.757673 12309 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.757741 15016 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.758313 12570 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.758852 16043 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.759258 12832 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.759371 12439 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.759619 16043 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.759995 15901 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.760699 15820 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.761289 12964 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.761818 12702 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:39.761940 16044 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.762559 16043 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.765797 15901 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:39.790150 15969 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.790830 12439 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.791128 12962 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.792126 12832 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.797030 15016 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.797523 12161 catalog_manager.cc:5484] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.796211 12570 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.802347 12702 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.805661 12309 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.806424 13094 consensus_queue.cc:237] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:39.809952 12832 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.810447 15016 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.810436 15901 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.810647 12962 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.811389 15969 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.811882 12439 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.812286 12306 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.812325 16043 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.813689 16044 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.814236 16044 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.816437 12570 raft_consensus.cc:1261] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:39.817049 16044 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.818363 12306 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.819418 15901 consensus_queue.cc:1035] T f8526552032e4a45ab4be4743f12abd0 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:39.821789 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f8526552032e4a45ab4be4743f12abd0 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:39.821641 12440 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.824611 12166 catalog_manager.cc:5484] T f8526552032e4a45ab4be4743f12abd0 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.825407 12832 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.824450 15016 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.828063 12962 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.829979 12570 raft_consensus.cc:2932] T f8526552032e4a45ab4be4743f12abd0 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:39.838306 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet f8526552032e4a45ab4be4743f12abd0 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:39.838675 16026 tablet_replica.cc:330] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:39.838842 16026 raft_consensus.cc:2229] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:39.839205 16026 raft_consensus.cc:2258] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:39.840600 16026 ts_tablet_manager.cc:1875] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:39.846701 16026 ts_tablet_manager.cc:1888] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:39.846822 16026 log.cc:1196] T f8526552032e4a45ab4be4743f12abd0 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/f8526552032e4a45ab4be4743f12abd0
I20240502 14:17:39.847556 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet f8526552032e4a45ab4be4743f12abd0 (table rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]) successfully deleted
I20240502 14:17:39.940716 12570 consensus_queue.cc:237] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:39.943785 12962 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:39.944825 12832 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:39.944835 15788 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:39.945823 16019 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:39.946003 12702 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:39.948684 12306 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:39.949147 12440 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:39.950423 13094 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:39.951789 15788 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:39.952176 16062 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:39.952783 16019 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:39.959319 16061 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:39.967156 12964 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:39.970118 12702 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:39.966321 16062 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:39.973991 12164 catalog_manager.cc:5484] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:39.982302 12570 consensus_queue.cc:237] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:39.984189 12440 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
W20240502 14:17:39.986196 12506 consensus_peers.cc:485] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 00aebb2064324c3eac8145d1c455622c. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:39.987108 12964 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:39.987512 12832 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:39.987632 16019 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:39.988343 16019 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:39.989900 12700 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:39.989353 13094 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:39.993494 13093 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:39.994544 12439 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:39.995069 16060 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:39.995288 16019 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:39.995710 12306 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:39.995780 16019 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:39.997485 16060 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.000296 12832 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.000708 12306 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.002867 16019 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.009029 12152 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 00aebb2064324c3eac8145d1c455622c with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:40.009792 12702 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.011436 14746 raft_consensus.cc:1203] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:40.012408 12833 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.014197 12439 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.017835 13094 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.018304 12306 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.024375 12161 catalog_manager.cc:5484] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.025076 12964 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.287310 12309 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:40.290710 13094 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.291162 12833 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.291419 12570 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.292095 16093 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.292402 16008 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.292990 16093 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.293884 12702 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.295042 16093 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.001s
I20240502 14:17:40.311854 16093 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.314393 12439 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.314987 12964 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.319995 16098 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.322052 16098 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.343502 13094 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.347659 12161 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:40.348857 12964 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.349380 12439 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.354518 12702 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.364557 12833 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.366561 12570 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:40.373224 12309 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:40.377236 12702 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:40.378119 16101 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:40.382208 13094 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:40.382773 16100 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:40.383190 12832 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:40.383224 12570 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:40.384096 16101 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:40.384691 16101 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:40.388106 16101 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.389324 12439 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 14:17:40.390921 12243 consensus_peers.cc:485] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b6bcd8515d234027abfd01ea5dbb3ede. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:40.391259 16104 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:40.392539 12570 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.394250 12964 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:40.395206 16099 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:40.397346 12439 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.399082 15950 raft_consensus.cc:1050] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:40.400326 15950 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:40.401707 12154 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet b6bcd8515d234027abfd01ea5dbb3ede with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:40.403573 13094 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.404678 12162 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.406759 12702 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.412340 12832 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.416148 12964 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:40.434602 12439 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:40.434804 14746 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:40.435364 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:40.435077 12570 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:40.436080 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.003s
I20240502 14:17:40.441246 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:40.453953 12833 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:40.456146 12309 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:40.456888 12702 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:40.458521 16114 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:40.459262 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:40.459787 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:40.472395 13094 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:40.473279 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:40.474572 16111 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.475410 12700 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.477994 12833 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.484560 12164 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.485746 12309 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.497534 13094 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.497712 14746 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.501945 12569 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.509204 12439 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.516327 12964 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:40.519408 12569 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:40.519999 14746 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:40.520289 12833 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:40.520323 12309 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:40.520380 13094 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:40.520876 16111 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:40.521129 15950 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:40.521345 16123 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:40.522011 16114 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:40.522228 15841 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:40.524788 12439 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:40.525638 16123 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:40.526760 12833 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.526777 15950 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.527377 12309 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.531559 14746 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.535332 13094 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.536562 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9cd04934b5264d7a9643974c0386cac4 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:40.537365 12161 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:40.537447 12569 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.552795 12439 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:40.554760 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet 9cd04934b5264d7a9643974c0386cac4 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:40.562072 16134 tablet_replica.cc:330] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:40.562297 16134 raft_consensus.cc:2229] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:40.562757 16134 raft_consensus.cc:2258] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:40.563786 16134 ts_tablet_manager.cc:1875] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:40.567766 16135 ts_tablet_manager.cc:902] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:40.568838 16134 ts_tablet_manager.cc:1888] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:40.569056 16134 log.cc:1196] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/9cd04934b5264d7a9643974c0386cac4
I20240502 14:17:40.569236 16135 tablet_copy_client.cc:308] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:40.569824 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet 9cd04934b5264d7a9643974c0386cac4 (table rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]) successfully deleted
I20240502 14:17:40.569947 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet 00aebb2064324c3eac8145d1c455622c from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:45555)
I20240502 14:17:40.570132 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet 00aebb2064324c3eac8145d1c455622c from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:45555: session id = 95e58ca33b4048e2b000dfbf2b5a0186-00aebb2064324c3eac8145d1c455622c
I20240502 14:17:40.572400 12592 tablet_copy_source_session.cc:205] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:40.573657 16135 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:40.578722 16135 tablet_copy_client.cc:783] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:40.578990 16135 tablet_copy_client.cc:647] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:40.580605 16135 tablet_copy_client.cc:523] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:40.583364 16135 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:40.618333 16135 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:40.618785 16135 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:40.619102 16135 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.036s user 0.028s sys 0.007s
I20240502 14:17:40.619890 16135 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:40.620276 16135 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:40.620508 16135 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:40.621320 16135 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.003s sys 0.001s
I20240502 14:17:40.622110 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-00aebb2064324c3eac8145d1c455622c received from {username='slave'} at 127.27.220.137:45555
I20240502 14:17:40.622238 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-00aebb2064324c3eac8145d1c455622c on tablet 00aebb2064324c3eac8145d1c455622c with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:40.859565 16135 ts_tablet_manager.cc:902] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:40.860790 16135 tablet_copy_client.cc:308] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:46613
I20240502 14:17:40.861505 12329 tablet_copy_service.cc:140] P 70a425a77fe54cbcb9755f9c540fd367: Received BeginTabletCopySession request for tablet b6bcd8515d234027abfd01ea5dbb3ede from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:51903)
I20240502 14:17:40.861750 12329 tablet_copy_service.cc:161] P 70a425a77fe54cbcb9755f9c540fd367: Beginning new tablet copy session on tablet b6bcd8515d234027abfd01ea5dbb3ede from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:51903: session id = 95e58ca33b4048e2b000dfbf2b5a0186-b6bcd8515d234027abfd01ea5dbb3ede
I20240502 14:17:40.864291 12329 tablet_copy_source_session.cc:205] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:40.865993 16135 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:40.870663 16135 tablet_copy_client.cc:783] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:40.870853 16135 tablet_copy_client.cc:647] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:40.872448 16135 tablet_copy_client.cc:523] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:40.874588 16135 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:40.907907 16135 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:40.908308 16135 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:40.908551 16135 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.034s user 0.024s sys 0.007s
I20240502 14:17:40.909197 16135 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:40.909435 16135 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:40.909651 16135 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:40.910192 16135 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.000s sys 0.004s
I20240502 14:17:40.910931 12329 tablet_copy_service.cc:342] P 70a425a77fe54cbcb9755f9c540fd367: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-b6bcd8515d234027abfd01ea5dbb3ede received from {username='slave'} at 127.27.220.137:51903
I20240502 14:17:40.911077 12329 tablet_copy_service.cc:433] P 70a425a77fe54cbcb9755f9c540fd367: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-b6bcd8515d234027abfd01ea5dbb3ede on tablet b6bcd8515d234027abfd01ea5dbb3ede with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:40.981956 12569 consensus_queue.cc:237] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:40.983412 15016 raft_consensus.cc:1203] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:40.992161 12833 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.992316 12702 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.992764 16019 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.992925 13094 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.993525 16019 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.993973 12309 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.994364 12964 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:40.995110 16168 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.995666 16168 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.996194 16168 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:40.996443 12439 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:41.003548 16060 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.006886 16019 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:41.012341 12833 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:41.013468 12439 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:41.018865 12964 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:41.019416 12161 catalog_manager.cc:5484] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:41.022308 13094 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:41.028759 12569 consensus_queue.cc:237] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:41.029345 12309 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:41.041718 12964 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.042363 16177 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.042428 12700 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.042742 13094 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.043429 16168 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.043942 16168 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.046905 12700 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:41.050134 12833 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.050745 16177 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.053153 16168 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.057348 12152 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:41.059331 12833 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.059711 12964 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.064352 12161 catalog_manager.cc:5484] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 14:17:41.065328 12507 consensus_peers.cc:485] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 76ce6bc11a2e4d12bb781b6c43c8b498. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:41.065783 13094 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.076907 12306 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.078341 16177 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.080332 12306 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
W20240502 14:17:41.081444 12374 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.131:43727, user_credentials={real_user=slave}} blocked reactor thread for 44459.7us
I20240502 14:17:41.083369 12439 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.086114 16170 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.087822 12439 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.091338 12700 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.243563 15016 raft_consensus.cc:1203] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:41.378470 16177 raft_consensus.cc:1050] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:41.379508 16177 consensus_queue.cc:237] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:41.382972 15016 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.383061 12964 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.383507 12439 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.383617 16170 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.383996 16060 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.384300 16170 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.384816 13094 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.384862 12309 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.385296 12700 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.385843 16168 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.386332 16170 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.387488 12832 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.388059 16060 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.388450 16170 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.413159 16170 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.422981 12700 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.427152 12439 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.430130 12309 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.429181 15016 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.432121 12964 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.434430 13094 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.435225 12832 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.438809 12166 catalog_manager.cc:5484] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.446290 16225 ts_tablet_manager.cc:902] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:41.450239 12569 consensus_queue.cc:237] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:41.460094 15016 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.460526 12439 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.460567 12964 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.462400 16060 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.462677 13094 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.462663 16168 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.462857 16177 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.463215 16224 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.467981 12833 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.467999 12440 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.468741 12309 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.467379 16170 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.472358 16168 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.472604 16224 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.473922 12161 catalog_manager.cc:5484] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.474465 13094 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.478112 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet 00aebb2064324c3eac8145d1c455622c with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:41.478906 16228 tablet_replica.cc:330] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:41.479169 16228 raft_consensus.cc:2229] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:41.478673 12833 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.479835 16228 raft_consensus.cc:2258] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:41.480993 16228 ts_tablet_manager.cc:1875] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:41.481011 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 00aebb2064324c3eac8145d1c455622c with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:41.481885 15016 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.485359 12309 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.488713 12964 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.494709 16228 ts_tablet_manager.cc:1888] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:41.494931 16228 log.cc:1196] T 00aebb2064324c3eac8145d1c455622c P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/00aebb2064324c3eac8145d1c455622c
I20240502 14:17:41.495961 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet 00aebb2064324c3eac8145d1c455622c (table rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]) successfully deleted
I20240502 14:17:41.497624 16225 tablet_copy_client.cc:308] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:41.504272 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:41013)
I20240502 14:17:41.504401 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet 76ce6bc11a2e4d12bb781b6c43c8b498 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:41013: session id = 4c04a4a8d8864267be7f67c2abe98f84-76ce6bc11a2e4d12bb781b6c43c8b498
I20240502 14:17:41.506489 12592 tablet_copy_source_session.cc:205] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:41.507809 16225 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 76ce6bc11a2e4d12bb781b6c43c8b498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:41.513484 16225 tablet_copy_client.cc:783] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:41.513756 16225 tablet_copy_client.cc:647] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:41.515432 16225 tablet_copy_client.cc:523] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:41.517832 16225 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:41.551676 16225 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:41.552152 16225 tablet_bootstrap.cc:492] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:41.552451 16225 ts_tablet_manager.cc:1367] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.035s user 0.032s sys 0.000s
I20240502 14:17:41.553221 16225 raft_consensus.cc:348] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:41.553596 16225 raft_consensus.cc:729] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:41.553894 16225 consensus_queue.cc:260] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:41.554864 16225 ts_tablet_manager.cc:1398] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.002s user 0.002s sys 0.003s
I20240502 14:17:41.555514 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-76ce6bc11a2e4d12bb781b6c43c8b498 received from {username='slave'} at 127.27.220.136:41013
I20240502 14:17:41.555680 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-76ce6bc11a2e4d12bb781b6c43c8b498 on tablet 76ce6bc11a2e4d12bb781b6c43c8b498 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:41.631208 16099 raft_consensus.cc:1050] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:41.632092 16099 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:41.635046 15016 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.635231 12964 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.635248 13094 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.635812 16008 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.636325 12833 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.636935 16105 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.637185 16250 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.637435 12570 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.637826 12440 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.637944 16105 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.638460 16105 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.638804 12700 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:41.638976 16105 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.640971 16008 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:41.657240 16252 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.662200 12964 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.666586 12440 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.667088 12164 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.666723 15016 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.669492 13094 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.671206 12700 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.674494 12570 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.681092 12832 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.685241 12309 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:41.689543 12570 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.689594 12832 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.690093 16008 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.690387 15016 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.690521 16105 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.690971 13094 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.691583 16252 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.691725 12440 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.692445 16252 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.693133 16105 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.694939 16105 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.695773 12964 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:41.696753 16008 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:41.698400 12964 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.697743 12440 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.699195 12832 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.701097 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b6bcd8515d234027abfd01ea5dbb3ede with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:41.701545 15016 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.702980 12570 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.703513 13094 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:41.703946 12166 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:41.711248 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:41.711678 16228 tablet_replica.cc:330] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:41.711870 16228 raft_consensus.cc:2229] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:41.712356 16228 raft_consensus.cc:2258] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:41.713418 16228 ts_tablet_manager.cc:1875] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:41.719633 16228 ts_tablet_manager.cc:1888] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:41.719789 16228 log.cc:1196] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/b6bcd8515d234027abfd01ea5dbb3ede
I20240502 14:17:41.720552 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet b6bcd8515d234027abfd01ea5dbb3ede (table rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]) successfully deleted
I20240502 14:17:41.799167 12570 consensus_queue.cc:237] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:41.801894 12440 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:41.801980 12309 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:41.802145 13094 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:41.802469 16168 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.802683 12964 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:41.802850 12700 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:41.803007 16060 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.803288 16168 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.803705 16168 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.806286 16060 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.806710 12832 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:41.808665 16060 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.809165 16060 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:41.810791 12964 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:41.814218 13094 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:41.815732 12164 catalog_manager.cc:5484] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:41.819519 12440 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:41.821663 12833 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:41.831070 12570 consensus_queue.cc:237] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
W20240502 14:17:41.835216 12507 consensus_peers.cc:485] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a6d444ae07234a30be060fbcc5b63b4f. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:41.836723 12964 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.837458 13094 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.837896 12306 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.837949 16224 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.838374 16224 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.838977 16170 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.839232 12702 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.840068 12306 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:41.841012 12440 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.841491 16224 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.841918 16168 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:41.843003 12702 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:41.844204 16168 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.846587 12832 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:41.847362 16224 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:41.848968 12152 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet a6d444ae07234a30be060fbcc5b63b4f with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:41.848827 12306 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.849383 12833 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.851555 13093 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.852792 12161 catalog_manager.cc:5484] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:41.856302 12439 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.856706 12702 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.858135 12964 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:41.915750 14746 raft_consensus.cc:1203] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:42.123685 13093 consensus_queue.cc:237] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } }
I20240502 14:17:42.126845 12833 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.126938 12702 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.127681 16112 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.127801 12570 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.128041 16052 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.129043 16052 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.130892 12306 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.131656 16295 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.132951 12964 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.134752 16295 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.134944 12439 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.136135 16052 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:42.138183 16112 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.141142 12702 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:42.142189 12306 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:42.142460 12570 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:42.143077 12162 catalog_manager.cc:5484] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } }
I20240502 14:17:42.146201 12964 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:42.155267 13093 consensus_queue.cc:237] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:42.159070 12306 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.159780 16304 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 14:17:42.161852 13030 consensus_peers.cc:485] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8b286a4283e9472980c6354935f99080. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:42.163560 12570 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.163673 12833 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.163756 12964 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.164093 12702 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.164222 16052 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.165122 16052 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.165587 12832 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:42.166260 16295 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.167029 16295 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.171592 16295 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:42.172246 12439 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } }
I20240502 14:17:42.174441 12702 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:42.176088 12832 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:42.177423 12154 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 8b286a4283e9472980c6354935f99080 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:42.177861 12440 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.178622 16112 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.178627 12964 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:42.180058 12440 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:42.180912 12161 catalog_manager.cc:5484] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:42.180858 12306 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:42.184518 12570 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:42.301261 16321 ts_tablet_manager.cc:902] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:42.314317 16321 tablet_copy_client.cc:308] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:42.315068 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet a6d444ae07234a30be060fbcc5b63b4f from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:41013)
I20240502 14:17:42.315248 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet a6d444ae07234a30be060fbcc5b63b4f from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:41013: session id = 4c04a4a8d8864267be7f67c2abe98f84-a6d444ae07234a30be060fbcc5b63b4f
I20240502 14:17:42.317325 12592 tablet_copy_source_session.cc:205] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:42.318451 16321 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:42.322499 16321 tablet_copy_client.cc:783] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:42.322700 16321 tablet_copy_client.cc:647] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:42.324362 16321 tablet_copy_client.cc:523] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:42.326659 16321 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:42.360066 16321 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:42.360503 16321 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:42.360812 16321 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.034s user 0.024s sys 0.008s
I20240502 14:17:42.361613 16321 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:42.361943 16321 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:42.362164 16321 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:42.362759 16321 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:17:42.363452 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-a6d444ae07234a30be060fbcc5b63b4f received from {username='slave'} at 127.27.220.136:41013
I20240502 14:17:42.363569 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-a6d444ae07234a30be060fbcc5b63b4f on tablet a6d444ae07234a30be060fbcc5b63b4f with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:42.486089 16224 raft_consensus.cc:1050] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:42.487183 16224 consensus_queue.cc:237] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:42.491813 13093 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:42.491909 14746 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:42.492473 16060 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:42.492785 16168 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:42.493924 12964 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:42.494671 16060 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:42.505991 12702 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:42.506247 12832 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:42.507313 16060 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:42.507889 16168 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:42.510449 12439 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:42.511065 16224 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:42.511351 12309 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:42.513111 16224 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:42.534547 16060 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.535863 12962 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.539278 12306 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.541117 12162 catalog_manager.cc:5484] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.545135 14746 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.571825 16351 ts_tablet_manager.cc:902] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:42.573602 13093 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.579878 12832 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.580688 12702 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.589812 12439 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.599153 12570 consensus_queue.cc:237] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:42.603169 13094 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:42.603338 14746 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:42.605288 12962 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:42.605319 12306 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:42.607806 12439 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:42.608526 16352 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:42.609251 16352 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:42.609763 12833 raft_consensus.cc:1261] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:42.609865 16352 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:42.610468 16350 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:42.612185 16351 tablet_copy_client.cc:308] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:42.612190 16352 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:42.612926 13116 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet 8b286a4283e9472980c6354935f99080 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:60037)
I20240502 14:17:42.613139 13116 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet 8b286a4283e9472980c6354935f99080 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:60037: session id = 1ba6a238e6ad4e4a84eff746ff336144-8b286a4283e9472980c6354935f99080
I20240502 14:17:42.613960 16168 consensus_queue.cc:1035] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:42.615895 13116 tablet_copy_source_session.cc:205] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:42.617944 16350 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.618988 16351 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:42.622279 12309 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.624847 16351 tablet_copy_client.cc:783] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:42.624723 13093 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.625581 16351 tablet_copy_client.cc:647] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:42.630070 12162 catalog_manager.cc:5484] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.634099 12439 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.635092 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:42.635546 16351 tablet_copy_client.cc:523] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:42.640130 12964 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.642252 16351 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:42.643537 14746 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.645236 12832 raft_consensus.cc:2932] T 76ce6bc11a2e4d12bb781b6c43c8b498 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:42.655540 16360 tablet_replica.cc:330] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:42.655915 16360 raft_consensus.cc:2229] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:42.656302 16360 raft_consensus.cc:2258] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:42.657243 16360 ts_tablet_manager.cc:1875] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:42.657948 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 76ce6bc11a2e4d12bb781b6c43c8b498 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:42.669651 16360 ts_tablet_manager.cc:1888] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:42.669915 16360 log.cc:1196] T 76ce6bc11a2e4d12bb781b6c43c8b498 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/76ce6bc11a2e4d12bb781b6c43c8b498
I20240502 14:17:42.670943 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet 76ce6bc11a2e4d12bb781b6c43c8b498 (table rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]) successfully deleted
I20240502 14:17:42.721931 16351 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:42.722462 16351 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:42.722770 16351 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.081s user 0.044s sys 0.008s
I20240502 14:17:42.723388 16351 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:42.723718 16351 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:42.723944 16351 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:42.724820 16351 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 14:17:42.725580 13116 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-8b286a4283e9472980c6354935f99080 received from {username='slave'} at 127.27.220.138:60037
I20240502 14:17:42.725715 13116 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-8b286a4283e9472980c6354935f99080 on tablet 8b286a4283e9472980c6354935f99080 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:42.736366 14746 raft_consensus.cc:1203] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:42.902547 12964 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:42.906103 12440 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.906688 13093 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.906719 16374 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.907008 12309 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.907135 12832 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.907753 12569 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.907802 16374 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.908304 16128 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.908855 12702 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:42.908696 16375 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.917865 16128 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.919482 16374 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:42.930194 16378 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:42.930860 12833 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:42.932060 13093 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:42.932628 12309 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:42.936512 12569 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:42.937245 12167 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:42.948323 12440 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:42.949421 12700 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:42.953814 12964 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:42.958289 12833 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 14:17:42.959028 12900 consensus_peers.cc:485] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b830809623f543438208dc87476d18c6. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:42.959633 12700 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.959981 16385 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.960147 12569 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.960896 16128 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.961287 16128 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.962468 13093 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.963042 12440 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.963492 16377 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.964329 16128 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.959700 12309 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:42.966707 16377 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:42.967218 16378 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:42.968390 12700 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:42.968293 13093 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:42.969151 12440 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:42.970676 12309 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:42.975661 12167 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:42.979904 12153 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet b830809623f543438208dc87476d18c6 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:42.980579 12832 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:42.982865 12569 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:43.030064 14883 raft_consensus.cc:1203] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:43.202574 16353 raft_consensus.cc:1050] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:43.203536 16353 consensus_queue.cc:237] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:43.206970 14746 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.207372 12309 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.207435 12440 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.208158 16060 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.209069 16353 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
I20240502 14:17:43.209389 16350 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.211503 12832 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.212301 16060 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.221007 13093 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.221891 16353 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.222101 12700 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.224898 12964 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.225641 16350 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.229794 16350 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.230520 16350 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.232117 12440 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.236105 12964 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.236380 12832 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.238423 14746 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.241565 12161 catalog_manager.cc:5484] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:43.244936 12309 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.256815 12569 consensus_queue.cc:237] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:43.261189 13093 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.261981 12832 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.262550 16403 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.266016 12309 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.266610 12962 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.267128 13094 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.267257 16353 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.268980 16353 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.270454 16403 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.271023 12439 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.272951 14745 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.274586 16405 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.276930 16353 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.275344 16405 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.278339 12309 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.278457 12440 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.283080 12832 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.284498 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a6d444ae07234a30be060fbcc5b63b4f with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:43.285490 12964 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.284693 14746 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.285843 13093 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:43.289208 12161 catalog_manager.cc:5484] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:43.295234 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet a6d444ae07234a30be060fbcc5b63b4f with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:43.306941 16421 tablet_replica.cc:330] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:43.307286 16421 raft_consensus.cc:2229] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:43.307612 16421 pending_rounds.cc:62] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: Trying to abort 1 pending ops.
I20240502 14:17:43.307716 16421 pending_rounds.cc:68] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: Aborting op as it isn't in flight: id { term: 1 index: 6 } timestamp: 7023245161278988288 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "a6d444ae07234a30be060fbcc5b63b4f" old_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } } new_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } } }
I20240502 14:17:43.308295 16421 raft_consensus.cc:2894] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Aborting config change with OpId 1.6: Aborted: Op aborted
I20240502 14:17:43.308475 16421 raft_consensus.cc:2866] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 14:17:43.308595 16421 raft_consensus.cc:2258] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:43.309614 16421 ts_tablet_manager.cc:1875] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:43.316462 16421 ts_tablet_manager.cc:1888] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:43.316619 16421 log.cc:1196] T a6d444ae07234a30be060fbcc5b63b4f P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/a6d444ae07234a30be060fbcc5b63b4f
I20240502 14:17:43.317414 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet a6d444ae07234a30be060fbcc5b63b4f (table rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]) successfully deleted
I20240502 14:17:43.446344 16430 ts_tablet_manager.cc:902] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:43.447700 16430 tablet_copy_client.cc:308] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:43.448398 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet b830809623f543438208dc87476d18c6 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:58685)
I20240502 14:17:43.448580 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet b830809623f543438208dc87476d18c6 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:58685: session id = 4c04a4a8d8864267be7f67c2abe98f84-b830809623f543438208dc87476d18c6
I20240502 14:17:43.450758 12985 tablet_copy_source_session.cc:205] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:43.452139 16430 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:43.457790 16430 tablet_copy_client.cc:783] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:43.458052 16430 tablet_copy_client.cc:647] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:43.459594 16430 tablet_copy_client.cc:523] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:43.461922 16430 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:43.494257 16430 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:43.494666 16430 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:43.494951 16430 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.033s user 0.025s sys 0.006s
I20240502 14:17:43.495723 16430 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:43.496066 16430 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:43.496229 16430 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:43.496737 16430 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:43.497880 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-b830809623f543438208dc87476d18c6 received from {username='slave'} at 127.27.220.136:58685
I20240502 14:17:43.498009 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-b830809623f543438208dc87476d18c6 on tablet b830809623f543438208dc87476d18c6 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:43.543108 16442 raft_consensus.cc:1050] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:43.544441 16442 consensus_queue.cc:237] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:43.549103 14883 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.548954 12440 consensus_queue.cc:237] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } }
I20240502 14:17:43.549693 12700 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.550058 16303 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.550137 12964 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.550655 16303 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.552596 12309 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:43.553438 16362 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:43.556711 12569 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:43.557577 16362 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:43.558049 16444 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.559911 12832 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.560571 16446 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.568473 12309 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.568545 12570 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.569293 16303 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.569965 16303 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.582278 12439 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:43.582974 16449 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:43.602584 14885 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.607182 13093 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:43.607496 12964 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:43.608091 16362 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:43.608559 16447 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:43.610169 12833 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:43.612116 16362 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:43.601778 16442 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.623576 16362 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:43.631233 13093 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:43.638325 12309 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.638916 12161 catalog_manager.cc:5484] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:43.642524 12962 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.646503 12832 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.653956 12569 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.653553 12700 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.658732 12164 catalog_manager.cc:5484] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.663365 12832 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:43.667690 12439 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.671948 12570 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:43.676669 12702 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 14:17:43.677760 13093 consensus_queue.cc:237] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:43.677745 12964 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:43.678885 16464 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 14:17:43.681195 12833 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.681591 12569 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.682607 16449 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.684568 14885 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.684988 12962 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.685215 16449 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.685678 12309 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.685742 16449 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.686667 16449 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.687666 16446 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.693352 16303 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.695307 12439 consensus_queue.cc:237] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:43.695672 12833 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.695996 12440 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:43.698825 12306 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
W20240502 14:17:43.699762 12375 consensus_peers.cc:485] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f358e8fda9ca4eeeb1dee5b2011ac029. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:43.702000 12834 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:43.702606 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8b286a4283e9472980c6354935f99080 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:43.702247 12569 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.703104 13093 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:43.703434 16449 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:43.703907 14885 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.704981 12964 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.706400 16466 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:43.707085 12962 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:43.707741 16466 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:43.709888 12307 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:43.711359 16466 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:43.712164 12162 catalog_manager.cc:5484] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.713433 12572 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:43.714023 16466 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:43.715425 16470 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:43.717264 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet 8b286a4283e9472980c6354935f99080 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:43.717725 16421 tablet_replica.cc:330] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:43.717921 16421 raft_consensus.cc:2229] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:43.718175 16421 raft_consensus.cc:2258] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:43.718181 16362 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:43.719586 16421 ts_tablet_manager.cc:1875] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:43.720404 12309 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.723798 12153 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet f358e8fda9ca4eeeb1dee5b2011ac029 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 14:17:43.717350 12439 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:43.724552 12834 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:43.724669 12307 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:43.725153 12572 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:43.728474 16421 ts_tablet_manager.cc:1888] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:43.731817 12162 catalog_manager.cc:5484] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:43.733502 16421 log.cc:1196] T 8b286a4283e9472980c6354935f99080 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/8b286a4283e9472980c6354935f99080
I20240502 14:17:43.737723 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet 8b286a4283e9472980c6354935f99080 (table rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]) successfully deleted
I20240502 14:17:43.739758 12962 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:43.740289 13093 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:43.746037 12702 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } }
I20240502 14:17:43.751322 12700 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 14:17:43.756599 16464 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 14:17:43.759291 12702 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:44.062060 14746 raft_consensus.cc:1203] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:44.220254 16497 ts_tablet_manager.cc:902] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:44.222404 16497 tablet_copy_client.cc:308] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:34491
I20240502 14:17:44.223079 12461 tablet_copy_service.cc:140] P eddeff583cf34c8dab2942fb36abbf9e: Received BeginTabletCopySession request for tablet f358e8fda9ca4eeeb1dee5b2011ac029 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:55391)
I20240502 14:17:44.223229 12461 tablet_copy_service.cc:161] P eddeff583cf34c8dab2942fb36abbf9e: Beginning new tablet copy session on tablet f358e8fda9ca4eeeb1dee5b2011ac029 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:55391: session id = 1ba6a238e6ad4e4a84eff746ff336144-f358e8fda9ca4eeeb1dee5b2011ac029
I20240502 14:17:44.226240 12461 tablet_copy_source_session.cc:205] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:44.227708 16497 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:44.232822 16497 tablet_copy_client.cc:783] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:44.233150 16497 tablet_copy_client.cc:647] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:44.234822 16497 tablet_copy_client.cc:523] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:44.237165 16497 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:44.271468 16497 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:44.271818 16497 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:44.272110 16497 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.035s user 0.030s sys 0.004s
I20240502 14:17:44.272742 16497 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:44.272991 16497 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:44.273188 16497 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:44.273936 16497 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:17:44.274600 12461 tablet_copy_service.cc:342] P eddeff583cf34c8dab2942fb36abbf9e: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-f358e8fda9ca4eeeb1dee5b2011ac029 received from {username='slave'} at 127.27.220.138:55391
I20240502 14:17:44.274736 12461 tablet_copy_service.cc:433] P eddeff583cf34c8dab2942fb36abbf9e: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-f358e8fda9ca4eeeb1dee5b2011ac029 on tablet f358e8fda9ca4eeeb1dee5b2011ac029 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:44.496572 16375 raft_consensus.cc:1050] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:44.497714 16375 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:44.500720 12572 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:44.500720 14745 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:44.501209 16495 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:44.501482 12439 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:44.501626 16385 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:44.502143 16385 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:44.503471 12309 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:44.505084 12832 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:44.505249 16385 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:44.505295 12702 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:44.505673 16495 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:44.506475 13094 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:44.507073 16526 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
I20240502 14:17:44.507829 16385 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:44.523531 16495 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.524324 12439 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.529023 12702 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.530458 12167 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.530485 12832 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.530792 12572 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.529111 14745 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.537286 13094 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.537827 12309 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.551254 12965 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:44.554028 12572 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:44.554487 12833 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:44.554625 16525 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:44.554767 14745 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:44.555197 16525 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:44.555711 16526 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:44.556730 13094 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:44.557045 12439 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:44.557871 12309 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:44.558835 16526 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:44.559137 16538 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:44.559919 16525 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:44.560464 16525 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.562312 14745 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.564008 12439 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.563791 12309 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.565132 12572 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.565156 12833 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.568176 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b830809623f543438208dc87476d18c6 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:44.570838 12161 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.571653 13094 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:44.576606 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet b830809623f543438208dc87476d18c6 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:44.578840 16540 tablet_replica.cc:330] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:44.580142 16540 raft_consensus.cc:2229] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:44.580658 16540 raft_consensus.cc:2258] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:44.581861 16540 ts_tablet_manager.cc:1875] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:44.586246 16540 ts_tablet_manager.cc:1888] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:44.586360 16540 log.cc:1196] T b830809623f543438208dc87476d18c6 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/b830809623f543438208dc87476d18c6
I20240502 14:17:44.587051 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet b830809623f543438208dc87476d18c6 (table rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]) successfully deleted
I20240502 14:17:44.681737 14885 raft_consensus.cc:1203] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 14:17:45.158501 16546 raft_consensus.cc:1050] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:45.159720 16546 consensus_queue.cc:237] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:45.162547 12572 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:45.162720 12309 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:45.162714 14885 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:45.163390 16470 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:45.163707 16565 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:45.164103 16548 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:45.166005 12965 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:45.166651 16548 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:45.168083 13094 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:45.169615 16548 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:45.175209 16548 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.177502 12702 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:45.177088 14885 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.177829 12833 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 14:17:45.178387 16546 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:45.178658 16565 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 14:17:45.180348 12572 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.181066 12965 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.179456 13094 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.185393 12165 catalog_manager.cc:5484] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:45.189914 12702 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.195142 12309 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.208966 12833 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.663882 12440 consensus_queue.cc:237] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:45.666682 13094 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:45.666743 12965 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:45.666772 14885 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:45.667380 12307 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:45.667533 12570 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:45.667760 12833 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 14:17:45.668049 16567 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:45.668345 16548 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:45.668787 16470 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:45.669399 16565 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:45.670212 16546 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:45.670527 16566 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 14:17:45.674537 16567 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.675530 12833 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.677261 14885 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.678845 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f358e8fda9ca4eeeb1dee5b2011ac029 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:45.679740 13094 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.680212 12570 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.682544 12962 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.686367 12307 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:45.686947 12164 catalog_manager.cc:5484] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 6 to 7, VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) evicted. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:45.696208 12682 tablet_service.cc:1514] Processing DeleteTablet for tablet f358e8fda9ca4eeeb1dee5b2011ac029 with delete_type TABLET_DATA_TOMBSTONED (TS af704d4ab2f2496a833c692bd3d25fde not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:47910
I20240502 14:17:45.700758 16592 tablet_replica.cc:330] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: stopping tablet replica
I20240502 14:17:45.701006 16592 raft_consensus.cc:2229] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:45.701287 16592 raft_consensus.cc:2258] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:45.702244 16592 ts_tablet_manager.cc:1875] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:45.707755 16592 ts_tablet_manager.cc:1888] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 14:17:45.707912 16592 log.cc:1196] T f358e8fda9ca4eeeb1dee5b2011ac029 P af704d4ab2f2496a833c692bd3d25fde: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-3/wal/wals/f358e8fda9ca4eeeb1dee5b2011ac029
I20240502 14:17:45.708678 12154 catalog_manager.cc:4830] TS af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): tablet f358e8fda9ca4eeeb1dee5b2011ac029 (table rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]) successfully deleted
I20240502 14:17:46.075508 14865 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:46.101238 12552 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:17:46.106828 14724 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:46.151347 13074 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:46.151767 12287 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:46.152062 14996 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:46.191290 12682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:46.202667 12419 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:17:46.210608 12813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:46.244729 12944 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:46.488691 14865 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:46.504597 14724 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:46.505832 12552 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:17:46.525100 12287 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:46.598182 14996 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:46.611945 12682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:46.612339 13074 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:46.633312 12944 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:46.634238 12419 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:17:46.641572 12813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:46.781447 12833 consensus_queue.cc:237] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:46.784742 12572 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.784970 14885 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.785161 12309 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.785298 16161 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.785913 16161 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.786073 12440 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.786572 16685 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.786983 16685 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.788972 13093 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.790100 16696 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.812500 12962 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.812573 16696 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:46.813388 12440 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:46.814142 14885 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:46.821910 12159 catalog_manager.cc:5484] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:46.824358 16161 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.826758 12962 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:46.830298 12572 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
W20240502 14:17:46.838135 12768 consensus_peers.cc:485] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b33c5e313e784b97987eb07b04f615f8. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:46.846459 12307 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:46.853842 13093 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:46.877151 13093 consensus_queue.cc:237] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:46.880360 12833 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.880978 16444 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.881700 14885 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.882253 12962 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.884320 16579 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.885023 16579 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 14:17:46.886059 13031 consensus_peers.cc:485] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8b286a4283e9472980c6354935f99080. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:46.902760 16579 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:46.903518 12572 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.904141 12307 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.904397 16444 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.904968 16444 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.906639 12440 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.907282 16687 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.908502 12962 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:46.915136 14885 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:46.919391 12164 catalog_manager.cc:5484] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:46.925737 12572 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:46.929780 12307 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:46.934767 12833 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:46.958886 12440 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:46.962148 13094 consensus_queue.cc:237] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
W20240502 14:17:46.966233 13030 consensus_peers.cc:485] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 01ffac38da1d4ece9cba649a59d5518b. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:46.966357 12572 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.966903 16444 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.969010 12307 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.969686 12702 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.970180 12439 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.970810 16579 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.971248 14745 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.971596 12833 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:46.972223 16716 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.972759 16716 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.973233 16716 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:46.973922 16687 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.002307 16579 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.007819 12700 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.015620 12440 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.017089 12833 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.019636 12307 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.019627 14745 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.024838 12161 catalog_manager.cc:5484] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.029048 12570 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.059048 12307 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:47.062901 15015 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.063688 16746 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.064823 13094 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.065448 12833 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.065707 12570 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.066236 16750 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.066635 16539 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.067584 12440 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.067584 12962 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.067574 16749 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.069628 16749 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.070262 16749 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 14:17:47.075944 12243 consensus_peers.cc:485] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b6bcd8515d234027abfd01ea5dbb3ede. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:47.079408 16746 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.084604 13093 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.085695 12962 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.086226 12440 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.089352 15015 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.090584 12164 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.094260 12833 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.100394 12570 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.123281 13093 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:47.126902 14885 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.127516 12962 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.127545 12570 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.128070 12833 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.128373 12307 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.128417 16736 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.128898 16579 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 14:17:47.129382 13031 consensus_peers.cc:485] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: df48755615874edab65577e92249fc78. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:47.129446 16579 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.129730 16444 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.130117 12440 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.130050 16723 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.132966 16716 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.136349 16723 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.137022 12570 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.141638 14885 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.141976 12962 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.143996 12161 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.156747 12440 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.157048 12833 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.164021 12307 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.179797 12962 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:47.183493 12440 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.183900 12570 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.184412 14745 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.184388 16645 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.185078 16385 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.185621 16385 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.209620 12307 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.209808 12833 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.210247 13093 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 14:17:47.210673 12899 consensus_peers.cc:485] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b830809623f543438208dc87476d18c6. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:47.210672 16645 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.211149 16385 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.211477 16645 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.218853 16645 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.221427 12440 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.233915 12162 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.247105 12307 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.255987 12570 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.263523 12832 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.272650 14745 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.272658 13094 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.285310 16788 ts_tablet_manager.cc:902] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069)
I20240502 14:17:47.310750 12962 consensus_queue.cc:237] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:47.315336 14745 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.315351 12570 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.316231 16645 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.316839 12833 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.317082 12439 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.317329 16645 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.317909 13094 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.318421 12307 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.319447 16645 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.320950 16781 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.321447 16645 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.324824 16788 tablet_copy_client.cc:308] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:45069
I20240502 14:17:47.332995 16777 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.336086 16787 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.337149 12307 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.339972 14745 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.339917 13093 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.344110 12833 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
W20240502 14:17:47.344926 12899 consensus_peers.cc:485] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 29809afc38e54766ba33a4e24311f8f4. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:47.346159 12164 catalog_manager.cc:5484] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:47.348011 12439 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
W20240502 14:17:47.368155 16796 log.cc:924] Time spent T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: Append to log took a long time: real 0.054s user 0.000s sys 0.000s
I20240502 14:17:47.371196 12570 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.378432 12854 tablet_copy_service.cc:140] P c78bd31d8d2142bf9d8e06b5a0074e0f: Received BeginTabletCopySession request for tablet b33c5e313e784b97987eb07b04f615f8 from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:36197)
I20240502 14:17:47.378609 12854 tablet_copy_service.cc:161] P c78bd31d8d2142bf9d8e06b5a0074e0f: Beginning new tablet copy session on tablet b33c5e313e784b97987eb07b04f615f8 from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:36197: session id = 95e58ca33b4048e2b000dfbf2b5a0186-b33c5e313e784b97987eb07b04f615f8
I20240502 14:17:47.381538 12854 tablet_copy_source_session.cc:205] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:47.383574 16788 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b33c5e313e784b97987eb07b04f615f8. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:47.387648 12439 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:47.391054 16788 tablet_copy_client.cc:783] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:47.391328 16788 tablet_copy_client.cc:647] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:47.394114 16788 tablet_copy_client.cc:523] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:47.394420 14885 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.394944 16688 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.396750 12962 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.396941 13093 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.397289 16567 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.397881 16567 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.399116 16788 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
W20240502 14:17:47.452149 12376 consensus_peers.cc:485] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3b3082d181004353bdff4414cd264d7c. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:47.455350 12833 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.455004 16815 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.455904 16819 ts_tablet_manager.cc:902] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:47.456149 16567 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.455487 12962 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.461621 12570 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.463157 12164 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.464892 12307 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.464754 13093 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.467875 16688 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.468478 16688 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 14:17:47.513506 12375 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.134:36329, user_credentials={real_user=slave}} blocked reactor thread for 52843.6us
W20240502 14:17:47.517270 12768 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.134:36329, user_credentials={real_user=slave}} blocked reactor thread for 58041.3us
I20240502 14:17:47.520438 12309 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.520869 14885 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.524772 16819 tablet_copy_client.cc:308] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:47.526386 13116 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet 8b286a4283e9472980c6354935f99080 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:52275)
I20240502 14:17:47.526597 13116 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet 8b286a4283e9472980c6354935f99080 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:52275: session id = 4c04a4a8d8864267be7f67c2abe98f84-8b286a4283e9472980c6354935f99080
I20240502 14:17:47.528229 12570 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.530495 12833 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:47.537851 13116 tablet_copy_source_session.cc:205] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:47.539646 16819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8b286a4283e9472980c6354935f99080. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:47.553158 16819 tablet_copy_client.cc:783] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:47.553572 16819 tablet_copy_client.cc:647] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:47.557624 16819 tablet_copy_client.cc:523] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:47.562060 16819 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:47.563448 12962 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:47.570658 14745 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.570668 12833 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.570776 13094 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.571573 12439 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.571625 16777 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.572160 16781 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.572252 12570 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 14:17:47.572533 12899 consensus_peers.cc:485] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9cd04934b5264d7a9643974c0386cac4. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:47.572989 12309 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.574687 16781 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.576614 16645 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.577119 16645 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.577714 16645 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.586864 16838 ts_tablet_manager.cc:902] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:47.596938 16837 ts_tablet_manager.cc:902] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:47.611063 16788 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:47.616003 16837 tablet_copy_client.cc:308] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:47.617488 13116 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet 01ffac38da1d4ece9cba649a59d5518b from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:46251)
I20240502 14:17:47.617744 13116 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet 01ffac38da1d4ece9cba649a59d5518b from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:46251: session id = 95e58ca33b4048e2b000dfbf2b5a0186-01ffac38da1d4ece9cba649a59d5518b
I20240502 14:17:47.619800 16777 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.622685 16788 tablet_bootstrap.cc:492] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:47.624406 16788 ts_tablet_manager.cc:1367] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.227s user 0.062s sys 0.007s
I20240502 14:17:47.623533 12307 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.626010 16788 raft_consensus.cc:348] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:47.627393 13116 tablet_copy_source_session.cc:205] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:47.633154 12162 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.638830 14746 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.639580 12833 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.640602 16837 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 01ffac38da1d4ece9cba649a59d5518b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:47.646481 12440 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.647325 13093 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.655881 12570 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:47.630004 16788 raft_consensus.cc:729] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:47.661432 16788 consensus_queue.cc:260] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:47.663338 16837 tablet_copy_client.cc:783] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:47.669800 16837 tablet_copy_client.cc:647] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:47.671983 16837 tablet_copy_client.cc:523] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:47.673388 16788 ts_tablet_manager.cc:1398] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.049s user 0.003s sys 0.003s
I20240502 14:17:47.675879 16837 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:47.681438 12854 tablet_copy_service.cc:342] P c78bd31d8d2142bf9d8e06b5a0074e0f: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-b33c5e313e784b97987eb07b04f615f8 received from {username='slave'} at 127.27.220.137:36197
I20240502 14:17:47.681663 12854 tablet_copy_service.cc:433] P c78bd31d8d2142bf9d8e06b5a0074e0f: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-b33c5e313e784b97987eb07b04f615f8 on tablet b33c5e313e784b97987eb07b04f615f8 with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:47.694687 16838 tablet_copy_client.cc:308] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:47.695487 13116 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet df48755615874edab65577e92249fc78 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:52275)
I20240502 14:17:47.695705 13116 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet df48755615874edab65577e92249fc78 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:52275: session id = 4c04a4a8d8864267be7f67c2abe98f84-df48755615874edab65577e92249fc78
I20240502 14:17:47.699779 13116 tablet_copy_source_session.cc:205] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:47.702271 16838 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:47.707257 16857 ts_tablet_manager.cc:902] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:47.707330 16819 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:47.707886 16819 tablet_bootstrap.cc:492] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:47.708266 16819 ts_tablet_manager.cc:1367] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.146s user 0.053s sys 0.005s
I20240502 14:17:47.711971 16819 raft_consensus.cc:348] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:47.712468 16819 raft_consensus.cc:729] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:47.713565 16838 tablet_copy_client.cc:783] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:47.712719 16819 consensus_queue.cc:260] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:47.713846 16838 tablet_copy_client.cc:647] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:47.714166 16819 ts_tablet_manager.cc:1398] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 14:17:47.716342 13116 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-8b286a4283e9472980c6354935f99080 received from {username='slave'} at 127.27.220.136:52275
I20240502 14:17:47.716511 13116 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-8b286a4283e9472980c6354935f99080 on tablet 8b286a4283e9472980c6354935f99080 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:47.719509 16838 tablet_copy_client.cc:523] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:47.723817 16838 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:47.724889 16857 tablet_copy_client.cc:308] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:46613
I20240502 14:17:47.730443 12833 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:47.757774 15015 raft_consensus.cc:1203] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:47.768174 16788 ts_tablet_manager.cc:902] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:47.772259 16837 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:47.772886 16837 tablet_bootstrap.cc:492] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:47.773248 16837 ts_tablet_manager.cc:1367] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.097s user 0.059s sys 0.018s
I20240502 14:17:47.774305 16837 raft_consensus.cc:348] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:47.774749 16837 raft_consensus.cc:729] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:47.775020 16837 consensus_queue.cc:260] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:47.775768 16837 ts_tablet_manager.cc:1398] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.001s sys 0.003s
I20240502 14:17:47.776633 13115 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-01ffac38da1d4ece9cba649a59d5518b received from {username='slave'} at 127.27.220.137:46251
I20240502 14:17:47.776768 13115 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-01ffac38da1d4ece9cba649a59d5518b on tablet 01ffac38da1d4ece9cba649a59d5518b with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:47.782372 14746 raft_consensus.cc:1203] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:47.791620 13094 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 14:17:47.792135 12768 consensus_peers.cc:485] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4ae8e0c47ce54724899b56aeb1b31336. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:47.793429 12700 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.794569 16824 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.795313 16824 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.797210 16870 ts_tablet_manager.cc:902] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:47.797744 12329 tablet_copy_service.cc:140] P 70a425a77fe54cbcb9755f9c540fd367: Received BeginTabletCopySession request for tablet b6bcd8515d234027abfd01ea5dbb3ede from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:34173)
I20240502 14:17:47.797945 12329 tablet_copy_service.cc:161] P 70a425a77fe54cbcb9755f9c540fd367: Beginning new tablet copy session on tablet b6bcd8515d234027abfd01ea5dbb3ede from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:34173: session id = 1ba6a238e6ad4e4a84eff746ff336144-b6bcd8515d234027abfd01ea5dbb3ede
I20240502 14:17:47.810308 14746 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.811218 16821 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.815358 16819 ts_tablet_manager.cc:902] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:47.826051 12329 tablet_copy_source_session.cc:205] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:47.828125 16857 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:47.836419 16857 tablet_copy_client.cc:783] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:47.836763 16857 tablet_copy_client.cc:647] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:47.843048 16857 tablet_copy_client.cc:523] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:47.847283 16857 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:47.868440 16824 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.869374 12570 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.870075 16821 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.878794 12700 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.889245 16819 tablet_copy_client.cc:308] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:34491
I20240502 14:17:47.894040 12309 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.894609 14746 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.896610 16788 tablet_copy_client.cc:308] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:47.897907 13093 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.897433 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet b830809623f543438208dc87476d18c6 from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:45165)
I20240502 14:17:47.898737 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet b830809623f543438208dc87476d18c6 from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:45165: session id = 95e58ca33b4048e2b000dfbf2b5a0186-b830809623f543438208dc87476d18c6
I20240502 14:17:47.902010 12985 tablet_copy_source_session.cc:205] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:47.902905 16872 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.903703 16788 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:47.905246 12440 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.905745 12570 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.908735 16821 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.910351 16788 tablet_copy_client.cc:783] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:47.910655 16788 tablet_copy_client.cc:647] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:47.912896 16788 tablet_copy_client.cc:523] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:47.923013 16788 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:47.924640 12162 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.930160 12440 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.933539 16870 tablet_copy_client.cc:308] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:47.935760 12309 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:47.945230 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet 29809afc38e54766ba33a4e24311f8f4 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:52341)
I20240502 14:17:47.945454 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet 29809afc38e54766ba33a4e24311f8f4 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:52341: session id = 1ba6a238e6ad4e4a84eff746ff336144-29809afc38e54766ba33a4e24311f8f4
I20240502 14:17:47.948657 12985 tablet_copy_source_session.cc:205] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:47.951839 16870 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 29809afc38e54766ba33a4e24311f8f4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:47.952246 12572 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:47.962783 15015 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.963476 16680 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.975334 13093 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.976346 16680 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.976550 12833 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.977169 16837 ts_tablet_manager.cc:902] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:47.977176 16353 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.987277 12962 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:47.988013 16353 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:47.999269 12440 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.000157 12309 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.000499 16353 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.001528 16353 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 14:17:48.004979 12507 consensus_peers.cc:485] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cd382683ec2b4600b6edaced79698b7e. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:48.012971 16353 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.013119 12439 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.019342 15015 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.020134 16870 tablet_copy_client.cc:783] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:48.020522 16870 tablet_copy_client.cc:647] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:48.030459 16870 tablet_copy_client.cc:523] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:48.032419 12164 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 reported cstate change: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.035609 12461 tablet_copy_service.cc:140] P eddeff583cf34c8dab2942fb36abbf9e: Received BeginTabletCopySession request for tablet 3b3082d181004353bdff4414cd264d7c from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:35863)
I20240502 14:17:48.035789 12461 tablet_copy_service.cc:161] P eddeff583cf34c8dab2942fb36abbf9e: Beginning new tablet copy session on tablet 3b3082d181004353bdff4414cd264d7c from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:35863: session id = 4c04a4a8d8864267be7f67c2abe98f84-3b3082d181004353bdff4414cd264d7c
I20240502 14:17:48.039202 12461 tablet_copy_source_session.cc:205] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:48.040568 16857 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:48.042894 16819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:48.048175 13094 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.051663 16819 tablet_copy_client.cc:783] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:48.051913 16819 tablet_copy_client.cc:647] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:48.053870 16857 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:48.053900 16819 tablet_copy_client.cc:523] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:48.054268 16857 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.207s user 0.052s sys 0.009s
I20240502 14:17:48.054853 16870 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:48.061434 12832 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.063818 15015 raft_consensus.cc:1203] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:48.055177 16857 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:48.064759 16857 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:48.065083 16857 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:48.065434 12307 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.067000 12962 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.067952 16837 tablet_copy_client.cc:308] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:48.068639 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet 9cd04934b5264d7a9643974c0386cac4 from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:45165)
I20240502 14:17:48.068789 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet 9cd04934b5264d7a9643974c0386cac4 from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:45165: session id = 95e58ca33b4048e2b000dfbf2b5a0186-9cd04934b5264d7a9643974c0386cac4
I20240502 14:17:48.071676 12985 tablet_copy_source_session.cc:205] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:48.073160 16837 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:48.088102 16857 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.034s user 0.002s sys 0.001s
I20240502 14:17:48.088955 12329 tablet_copy_service.cc:342] P 70a425a77fe54cbcb9755f9c540fd367: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-b6bcd8515d234027abfd01ea5dbb3ede received from {username='slave'} at 127.27.220.138:34173
I20240502 14:17:48.089112 12329 tablet_copy_service.cc:433] P 70a425a77fe54cbcb9755f9c540fd367: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-b6bcd8515d234027abfd01ea5dbb3ede on tablet b6bcd8515d234027abfd01ea5dbb3ede with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:48.090042 16819 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:48.091706 16837 tablet_copy_client.cc:783] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:48.092098 16837 tablet_copy_client.cc:647] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:48.094417 16837 tablet_copy_client.cc:523] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:48.097492 16838 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:48.101364 16837 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:48.102372 16838 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:48.103278 16838 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.380s user 0.058s sys 0.028s
I20240502 14:17:48.104697 16838 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:48.107383 16838 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:48.107805 16838 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:48.109040 16838 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240502 14:17:48.110241 13115 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-df48755615874edab65577e92249fc78 received from {username='slave'} at 127.27.220.136:52275
I20240502 14:17:48.110378 13115 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-df48755615874edab65577e92249fc78 on tablet df48755615874edab65577e92249fc78 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:48.141775 12572 consensus_queue.cc:237] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:48.143226 16788 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:48.151875 14885 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.152299 12439 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.152560 12307 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.152935 16353 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.155246 16353 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.157073 16353 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.157804 16788 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:48.158105 16788 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.235s user 0.051s sys 0.009s
I20240502 14:17:48.158926 16788 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:48.159236 16788 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:48.159468 16788 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:48.160066 16788 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 14:17:48.161891 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-b830809623f543438208dc87476d18c6 received from {username='slave'} at 127.27.220.137:45165
I20240502 14:17:48.162034 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-b830809623f543438208dc87476d18c6 on tablet b830809623f543438208dc87476d18c6 with peer 95e58ca33b4048e2b000dfbf2b5a0186
W20240502 14:17:48.164875 12506 consensus_peers.cc:485] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ddf70e508caa4f48a9255be579060d97. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:48.167984 13094 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.169100 12700 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.169214 12832 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.169940 16908 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.170501 16908 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.170979 16908 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.184026 15015 raft_consensus.cc:1203] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:48.189975 16837 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:48.190562 16837 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:48.190927 16837 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.090s user 0.053s sys 0.001s
I20240502 14:17:48.191022 16870 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:48.191984 16837 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:48.192489 16837 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:48.192783 16837 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:48.193646 16837 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:48.194506 16870 tablet_bootstrap.cc:492] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:48.194880 16870 ts_tablet_manager.cc:1367] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.140s user 0.060s sys 0.010s
I20240502 14:17:48.196389 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-9cd04934b5264d7a9643974c0386cac4 received from {username='slave'} at 127.27.220.137:45165
I20240502 14:17:48.196570 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-9cd04934b5264d7a9643974c0386cac4 on tablet 9cd04934b5264d7a9643974c0386cac4 with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:48.195802 16870 raft_consensus.cc:348] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:48.197101 16870 raft_consensus.cc:729] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:48.197436 16870 consensus_queue.cc:260] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:48.198320 16870 ts_tablet_manager.cc:1398] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 14:17:48.200572 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-29809afc38e54766ba33a4e24311f8f4 received from {username='slave'} at 127.27.220.138:52341
I20240502 14:17:48.200722 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-29809afc38e54766ba33a4e24311f8f4 on tablet 29809afc38e54766ba33a4e24311f8f4 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:48.214454 16908 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:48.215850 13094 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:48.219194 14885 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:48.222805 12700 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:48.223692 12439 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:48.228013 12161 catalog_manager.cc:5484] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:48.229105 12832 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:48.235669 12307 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:48.254268 14885 raft_consensus.cc:1203] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:48.260318 16819 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:48.261963 16819 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:48.262465 16819 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.173s user 0.057s sys 0.006s
I20240502 14:17:48.263314 16819 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:48.263672 16819 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:48.263912 16819 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:48.264560 16819 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:48.265398 12461 tablet_copy_service.cc:342] P eddeff583cf34c8dab2942fb36abbf9e: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-3b3082d181004353bdff4414cd264d7c received from {username='slave'} at 127.27.220.136:35863
I20240502 14:17:48.265564 12461 tablet_copy_service.cc:433] P eddeff583cf34c8dab2942fb36abbf9e: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-3b3082d181004353bdff4414cd264d7c on tablet 3b3082d181004353bdff4414cd264d7c with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:48.273176 12439 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:48.276772 14885 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 14:17:48.277328 12376 consensus_peers.cc:485] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c9ff45109a6d4dc9aba4617819929981. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:48.277534 12572 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.277606 16825 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.278682 16567 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.278800 12307 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.278898 13094 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.279722 16567 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.280201 16567 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.281164 12832 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.282058 12700 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:48.282060 16567 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.283638 16567 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:48.295773 16825 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.301211 13093 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.308555 14883 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.310096 16821 raft_consensus.cc:1050] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:48.309985 12162 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:48.311203 16821 consensus_queue.cc:237] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:48.312768 12572 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.313920 12307 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.316476 16870 ts_tablet_manager.cc:902] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069)
I20240502 14:17:48.321148 16723 raft_consensus.cc:1050] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:48.322616 16723 consensus_queue.cc:237] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:48.328670 14885 raft_consensus.cc:1203] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:48.335855 12700 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.342149 12572 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.342232 12439 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.342690 16872 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.343062 16696 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.343632 14885 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.344410 16824 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.344796 15015 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.345721 12307 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.346582 16824 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.347784 14746 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.350603 16872 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.354108 12832 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:48.367671 16870 tablet_copy_client.cc:308] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:45069
I20240502 14:17:48.370363 12854 tablet_copy_service.cc:140] P c78bd31d8d2142bf9d8e06b5a0074e0f: Received BeginTabletCopySession request for tablet 4ae8e0c47ce54724899b56aeb1b31336 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:49687)
I20240502 14:17:48.370605 12854 tablet_copy_service.cc:161] P c78bd31d8d2142bf9d8e06b5a0074e0f: Beginning new tablet copy session on tablet 4ae8e0c47ce54724899b56aeb1b31336 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:49687: session id = 1ba6a238e6ad4e4a84eff746ff336144-4ae8e0c47ce54724899b56aeb1b31336
I20240502 14:17:48.372613 13094 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.373359 16947 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.382704 14746 raft_consensus.cc:1203] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:48.392468 12962 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.393246 16821 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.401187 12854 tablet_copy_source_session.cc:205] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:48.415971 16872 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.416750 12307 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.419752 16444 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.423105 12162 catalog_manager.cc:5484] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.430364 12833 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.430390 12965 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.432242 12439 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.433069 16444 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.435017 14885 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.437041 16870 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:48.438299 15015 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.439327 12834 consensus_queue.cc:237] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:48.432484 13094 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.444880 12965 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.445165 12307 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.446331 16947 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.446730 16821 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.448474 14884 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.458662 16716 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.460348 12440 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.461447 16947 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.465258 12307 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.466151 16716 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.468173 16870 tablet_copy_client.cc:783] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:48.468248 12570 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.468569 16870 tablet_copy_client.cc:647] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:48.468747 16947 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.469872 16964 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.473816 14745 raft_consensus.cc:1203] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:48.476639 12441 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.477986 12155 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b33c5e313e784b97987eb07b04f615f8 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:48.478389 15016 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.479209 12307 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.481367 13093 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.482204 16872 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.483959 13093 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.485656 16696 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.485639 12162 catalog_manager.cc:5484] T b33c5e313e784b97987eb07b04f615f8 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.493249 16819 ts_tablet_manager.cc:902] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:48.493402 12552 tablet_service.cc:1514] Processing DeleteTablet for tablet b33c5e313e784b97987eb07b04f615f8 with delete_type TABLET_DATA_TOMBSTONED (TS 53479f0fb09c454e9b3cbe7398bf8874 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:59642
I20240502 14:17:48.497913 12962 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.503186 12441 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.503876 12965 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.508250 16964 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.509027 14883 raft_consensus.cc:1261] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.511222 16872 consensus_queue.cc:1035] T b33c5e313e784b97987eb07b04f615f8 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.512616 16964 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.513753 14883 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.522122 15015 raft_consensus.cc:2932] T b33c5e313e784b97987eb07b04f615f8 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.526445 16964 raft_consensus.cc:1050] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:48.527904 16964 consensus_queue.cc:237] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:48.530397 16970 tablet_replica.cc:330] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: stopping tablet replica
I20240502 14:17:48.531291 16970 raft_consensus.cc:2229] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:48.532245 16970 pending_rounds.cc:62] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Trying to abort 1 pending ops.
I20240502 14:17:48.539752 16962 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.548416 14883 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.551990 16870 tablet_copy_client.cc:523] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:48.556335 16870 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:48.556257 12161 catalog_manager.cc:5484] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.561321 15014 raft_consensus.cc:1203] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:48.562954 16970 pending_rounds.cc:68] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Aborting op as it isn't in flight: id { term: 1 index: 9 } timestamp: 7023245182199717888 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "b33c5e313e784b97987eb07b04f615f8" old_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } } new_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } } }
I20240502 14:17:48.569032 16970 raft_consensus.cc:2894] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Aborting config change with OpId 1.9: Aborted: Op aborted
I20240502 14:17:48.569252 16970 raft_consensus.cc:2866] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 14:17:48.569420 16970 raft_consensus.cc:2258] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:48.570241 12441 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.571697 16970 ts_tablet_manager.cc:1875] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:48.572683 14745 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.582000 12833 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.587881 16819 tablet_copy_client.cc:308] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:48.589123 12307 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.593216 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet cd382683ec2b4600b6edaced79698b7e from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:41013)
I20240502 14:17:48.593398 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet cd382683ec2b4600b6edaced79698b7e from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:41013: session id = 4c04a4a8d8864267be7f67c2abe98f84-cd382683ec2b4600b6edaced79698b7e
I20240502 14:17:48.596547 12592 tablet_copy_source_session.cc:205] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:48.598141 16819 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:48.598366 16970 ts_tablet_manager.cc:1888] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:48.598587 16970 log.cc:1196] T b33c5e313e784b97987eb07b04f615f8 P 53479f0fb09c454e9b3cbe7398bf8874: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/b33c5e313e784b97987eb07b04f615f8
I20240502 14:17:48.601199 12152 catalog_manager.cc:4830] TS 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): tablet b33c5e313e784b97987eb07b04f615f8 (table rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]) successfully deleted
I20240502 14:17:48.603538 15014 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.603690 14745 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.604506 12572 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.605018 12307 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.605235 16444 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.605815 16716 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.606514 16444 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.609788 12441 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.612304 16444 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.613427 16949 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.635593 12832 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.637073 12700 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.637735 16723 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.639616 16444 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.645877 16819 tablet_copy_client.cc:783] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:48.646210 16819 tablet_copy_client.cc:647] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:48.649966 16819 tablet_copy_client.cc:523] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:48.654119 16819 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:48.664278 16837 ts_tablet_manager.cc:902] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:48.665697 12962 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.674770 16645 raft_consensus.cc:1050] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:48.676008 16645 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:48.688256 14746 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.688915 12572 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.689474 12307 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.690956 12441 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.690567 16777 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.001s
I20240502 14:17:48.692430 16777 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.693648 16777 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.694051 16837 tablet_copy_client.cc:308] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:48.694311 16777 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.696054 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet ddf70e508caa4f48a9255be579060d97 from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:45555)
I20240502 14:17:48.696225 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet ddf70e508caa4f48a9255be579060d97 from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:45555: session id = 95e58ca33b4048e2b000dfbf2b5a0186-ddf70e508caa4f48a9255be579060d97
I20240502 14:17:48.699352 12592 tablet_copy_source_session.cc:205] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:48.700691 16837 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ddf70e508caa4f48a9255be579060d97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:48.704610 15015 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.705713 12834 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.705668 16777 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.706387 16777 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.706899 16957 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.719614 15015 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.719305 12569 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.720180 12832 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.724969 14745 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.728904 12162 catalog_manager.cc:5484] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.730139 12700 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.730953 12309 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.735114 16837 tablet_copy_client.cc:783] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:48.735395 16837 tablet_copy_client.cc:647] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
W20240502 14:17:48.740339 16870 log.cc:924] Time spent T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: Append to log took a long time: real 0.053s user 0.001s sys 0.000s
I20240502 14:17:48.743810 16837 tablet_copy_client.cc:523] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:48.747435 16837 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
W20240502 14:17:48.754022 12900 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:44203, user_credentials={real_user=slave}} blocked reactor thread for 62237.3us
I20240502 14:17:48.753495 12441 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.754945 16870 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:48.755405 16870 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:48.757273 13094 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.766759 13093 consensus_queue.cc:237] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:48.781152 16870 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.225s user 0.063s sys 0.005s
I20240502 14:17:48.782485 12307 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.782799 16870 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:48.783403 16444 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.783432 16870 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:48.784090 16870 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:48.786975 16870 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.006s user 0.003s sys 0.000s
I20240502 14:17:48.789819 15015 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.790694 16723 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.791765 12572 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.792448 16723 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.796767 16723 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.797942 16781 raft_consensus.cc:1050] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:48.796903 16645 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.799588 16781 consensus_queue.cc:237] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:48.803326 17005 ts_tablet_manager.cc:902] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:48.803660 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 01ffac38da1d4ece9cba649a59d5518b with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:48.805008 15015 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.805563 12307 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.806044 12569 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.806473 12441 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.812712 12162 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.813388 12161 catalog_manager.cc:5484] T 01ffac38da1d4ece9cba649a59d5518b P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.817406 16996 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.820626 12854 tablet_copy_service.cc:342] P c78bd31d8d2142bf9d8e06b5a0074e0f: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-4ae8e0c47ce54724899b56aeb1b31336 received from {username='slave'} at 127.27.220.138:49687
I20240502 14:17:48.820758 12854 tablet_copy_service.cc:433] P c78bd31d8d2142bf9d8e06b5a0074e0f: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-4ae8e0c47ce54724899b56aeb1b31336 on tablet 4ae8e0c47ce54724899b56aeb1b31336 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:48.823397 12419 tablet_service.cc:1514] Processing DeleteTablet for tablet 01ffac38da1d4ece9cba649a59d5518b with delete_type TABLET_DATA_TOMBSTONED (TS eddeff583cf34c8dab2942fb36abbf9e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:52872
I20240502 14:17:48.823125 14744 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.821297 12307 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.826879 12569 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.842252 12833 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.851199 16567 raft_consensus.cc:1050] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:48.852813 16567 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:48.854516 12570 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.857901 12831 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.858726 14746 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.859874 16964 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.860571 16964 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.862705 14746 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.862779 12833 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.863165 13093 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:48.872241 14883 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.873018 16815 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.884109 13096 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.884585 13094 consensus_queue.cc:237] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:48.885176 16815 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.889230 12962 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.889962 16815 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.891140 12833 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.891746 16964 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.892092 14746 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.892273 14885 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.892632 16964 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.893426 12965 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.893471 12441 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.893623 16964 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.894748 12570 raft_consensus.cc:1261] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.895411 16964 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.897297 16957 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.897639 16964 consensus_queue.cc:1035] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.899688 16949 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.906363 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8b286a4283e9472980c6354935f99080 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:48.906386 12569 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.907061 14885 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.913450 12441 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.918129 12167 catalog_manager.cc:5484] T 8b286a4283e9472980c6354935f99080 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.926318 14744 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.927268 16746 raft_consensus.cc:1050] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:48.928474 16964 raft_consensus.cc:1050] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:48.928458 16746 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:48.929617 16964 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:48.934051 12287 tablet_service.cc:1514] Processing DeleteTablet for tablet 8b286a4283e9472980c6354935f99080 with delete_type TABLET_DATA_TOMBSTONED (TS 70a425a77fe54cbcb9755f9c540fd367 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:48502
I20240502 14:17:48.935839 12570 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.936749 16815 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.943365 14745 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.944690 12307 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.945298 16815 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.946771 16815 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.960392 12832 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.965304 12834 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.967080 16567 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.976816 15015 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.977505 16930 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.979981 12441 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.980559 16930 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.980679 12570 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:48.981796 12702 raft_consensus.cc:1261] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:48.981787 16930 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:48.982653 16957 consensus_queue.cc:1035] T 01ffac38da1d4ece9cba649a59d5518b P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:48.984385 12702 raft_consensus.cc:2932] T 01ffac38da1d4ece9cba649a59d5518b P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.001605 12831 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.001966 12569 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.002506 16723 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.003041 16723 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.008280 12963 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.009773 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.016785 14884 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.017426 14883 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.017989 16962 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.018486 16930 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.021595 12309 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.022280 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.022769 12439 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.025152 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.031121 14744 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.032002 16962 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.045326 16825 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
W20240502 14:17:49.048285 17029 log.cc:924] Time spent T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e: Append to log took a long time: real 0.057s user 0.000s sys 0.000s
I20240502 14:17:49.059067 16964 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.062366 12569 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.071628 12167 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.076525 17005 tablet_copy_client.cc:308] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:34491
I20240502 14:17:49.086011 12309 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.077658 14744 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.095613 12441 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.097491 12461 tablet_copy_service.cc:140] P eddeff583cf34c8dab2942fb36abbf9e: Received BeginTabletCopySession request for tablet c9ff45109a6d4dc9aba4617819929981 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:35863)
I20240502 14:17:49.097692 12461 tablet_copy_service.cc:161] P eddeff583cf34c8dab2942fb36abbf9e: Beginning new tablet copy session on tablet c9ff45109a6d4dc9aba4617819929981 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:35863: session id = 4c04a4a8d8864267be7f67c2abe98f84-c9ff45109a6d4dc9aba4617819929981
I20240502 14:17:49.100845 12461 tablet_copy_source_session.cc:205] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:49.101961 14745 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.103374 12833 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.107203 17005 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:49.110839 12834 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.112448 16930 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.113019 13093 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.113994 16930 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
W20240502 14:17:49.116739 17013 log.cc:924] Time spent T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: Append to log took a long time: real 0.249s user 0.000s sys 0.000s
I20240502 14:17:49.116930 12831 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:48.906930 12965 raft_consensus.cc:2932] T 8b286a4283e9472980c6354935f99080 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.123708 12569 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.128351 16746 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.138844 12164 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.150430 17012 tablet_replica.cc:330] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: stopping tablet replica
I20240502 14:17:49.151041 17012 raft_consensus.cc:2229] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:49.151841 17012 raft_consensus.cc:2258] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:49.152853 17023 tablet_replica.cc:330] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: stopping tablet replica
I20240502 14:17:49.153529 17023 raft_consensus.cc:2229] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:49.153939 17023 raft_consensus.cc:2258] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:49.147320 14884 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.161644 12962 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.166180 15014 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.166510 14883 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.168917 16777 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.169854 12161 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:49.171684 14885 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.175632 17023 ts_tablet_manager.cc:1875] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:49.177002 14882 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.177265 12570 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.184485 17012 ts_tablet_manager.cc:1875] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:49.184423 13094 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.189476 12307 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.191924 16995 raft_consensus.cc:1050] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:49.193691 16995 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:49.194469 12309 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.196045 16777 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.197415 13096 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.198292 16777 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.201495 14746 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.202051 16781 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.209192 17012 ts_tablet_manager.cc:1888] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:49.213650 17012 log.cc:1196] T 01ffac38da1d4ece9cba649a59d5518b P eddeff583cf34c8dab2942fb36abbf9e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/01ffac38da1d4ece9cba649a59d5518b
I20240502 14:17:49.215083 12153 catalog_manager.cc:4830] TS eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491): tablet 01ffac38da1d4ece9cba649a59d5518b (table rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]) successfully deleted
I20240502 14:17:49.219297 13093 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
W20240502 14:17:49.228008 12507 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:44203, user_credentials={real_user=slave}} blocked reactor thread for 74224.5us
I20240502 14:17:49.228658 15015 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.231040 12441 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.231119 12569 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.232299 16781 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.236856 16781 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.239534 12441 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.225219 12833 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.247140 14744 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.247995 16781 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.252168 12309 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.252727 16777 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.254750 12570 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.256022 14884 raft_consensus.cc:1203] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:49.256561 16777 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.262645 12440 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.263466 16781 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.280862 16777 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.288298 13093 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.288691 12307 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.294387 14745 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.295140 12161 catalog_manager.cc:5484] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.298365 13093 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.299014 16781 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.300289 12439 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.307340 17005 tablet_copy_client.cc:783] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:49.307636 17005 tablet_copy_client.cc:647] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:49.314345 17005 tablet_copy_client.cc:523] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:49.317144 14883 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.326596 15015 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.328814 17052 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.342301 17023 ts_tablet_manager.cc:1888] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:49.342594 17023 log.cc:1196] T 8b286a4283e9472980c6354935f99080 P 70a425a77fe54cbcb9755f9c540fd367: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/8b286a4283e9472980c6354935f99080
I20240502 14:17:49.343722 12154 catalog_manager.cc:4830] TS 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): tablet 8b286a4283e9472980c6354935f99080 (table rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]) successfully deleted
I20240502 14:17:49.344362 16995 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.351653 17005 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:49.350934 12570 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.351954 12307 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.358253 12161 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.358464 12439 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.368846 16746 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.369772 14745 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.372583 12834 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.374632 12834 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.381821 13094 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:49.386826 14746 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.386854 14884 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.387363 16964 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.387823 12570 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.388425 16964 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.389173 16964 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.388535 12440 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.393484 16949 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.393216 16957 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.393529 13094 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.397732 12570 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.400668 15015 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.403453 12439 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.405558 12569 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.410566 12441 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:49.415352 12831 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.416301 13096 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.416631 16567 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.417066 17047 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.417678 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet df48755615874edab65577e92249fc78 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:49.418069 14743 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.419620 14746 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.420171 17047 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.422389 14884 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.423492 12164 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:49.421180 12963 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.432781 12287 tablet_service.cc:1514] Processing DeleteTablet for tablet df48755615874edab65577e92249fc78 with delete_type TABLET_DATA_TOMBSTONED (TS 70a425a77fe54cbcb9755f9c540fd367 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:48502
I20240502 14:17:49.434322 17023 tablet_replica.cc:330] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: stopping tablet replica
I20240502 14:17:49.434659 17023 raft_consensus.cc:2229] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:49.435125 17023 raft_consensus.cc:2258] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:49.434613 16567 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.437148 17023 ts_tablet_manager.cc:1875] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:49.440505 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3b3082d181004353bdff4414cd264d7c with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:49.441002 12834 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.442039 14746 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.442670 12831 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.443441 12569 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.444041 16567 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.446146 12569 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.446786 14883 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.447583 16962 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.450384 12162 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 reported cstate change: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.451121 17045 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.450999 13093 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.453823 14883 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.454947 12962 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.455478 12964 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.455986 12962 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.462123 12962 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.462188 16995 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.462972 16995 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.463019 12287 tablet_service.cc:1514] Processing DeleteTablet for tablet 3b3082d181004353bdff4414cd264d7c with delete_type TABLET_DATA_TOMBSTONED (TS 70a425a77fe54cbcb9755f9c540fd367 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:48502
I20240502 14:17:49.464669 16949 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.465107 17047 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.467120 12961 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.468160 12965 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.481105 12833 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:49.489269 12834 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.495997 12831 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.498009 12963 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:49.515277 12569 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.515609 12307 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.516005 16995 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.516628 14745 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.516716 17053 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.517311 16995 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.523348 17053 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.525405 15015 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.526021 16777 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.527029 12833 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.527870 13093 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.528252 17023 ts_tablet_manager.cc:1888] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:49.528220 17052 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.528409 17023 log.cc:1196] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/df48755615874edab65577e92249fc78
I20240502 14:17:49.527895 15015 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.529172 17052 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.529245 12154 catalog_manager.cc:4830] TS 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): tablet df48755615874edab65577e92249fc78 (table rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]) successfully deleted
I20240502 14:17:49.531091 13093 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.540763 12307 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.549104 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b830809623f543438208dc87476d18c6 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:49.550745 14746 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.551087 12569 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.542979 17023 tablet_replica.cc:330] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: stopping tablet replica
I20240502 14:17:49.559016 17023 raft_consensus.cc:2229] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:49.558564 12166 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 reported cstate change: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.559577 17023 raft_consensus.cc:2258] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:49.560922 17023 ts_tablet_manager.cc:1875] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:49.564929 12419 tablet_service.cc:1514] Processing DeleteTablet for tablet b830809623f543438208dc87476d18c6 with delete_type TABLET_DATA_TOMBSTONED (TS eddeff583cf34c8dab2942fb36abbf9e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:52872
I20240502 14:17:49.565438 17012 tablet_replica.cc:330] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: stopping tablet replica
I20240502 14:17:49.565745 17012 raft_consensus.cc:2229] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:49.566112 17012 raft_consensus.cc:2258] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:49.567400 17012 ts_tablet_manager.cc:1875] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:49.567332 12834 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:49.588156 17012 ts_tablet_manager.cc:1888] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:49.588405 17012 log.cc:1196] T b830809623f543438208dc87476d18c6 P eddeff583cf34c8dab2942fb36abbf9e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/b830809623f543438208dc87476d18c6
I20240502 14:17:49.589401 12153 catalog_manager.cc:4830] TS eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491): tablet b830809623f543438208dc87476d18c6 (table rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]) successfully deleted
I20240502 14:17:49.596374 17023 ts_tablet_manager.cc:1888] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:49.596539 17023 log.cc:1196] T 3b3082d181004353bdff4414cd264d7c P 70a425a77fe54cbcb9755f9c540fd367: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/3b3082d181004353bdff4414cd264d7c
I20240502 14:17:49.597432 12154 catalog_manager.cc:4830] TS 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): tablet 3b3082d181004353bdff4414cd264d7c (table rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]) successfully deleted
I20240502 14:17:49.626780 16837 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:49.627946 16837 tablet_bootstrap.cc:492] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:49.629561 16837 ts_tablet_manager.cc:1367] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.882s user 0.050s sys 0.010s
I20240502 14:17:49.630517 16837 raft_consensus.cc:348] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:49.630981 16837 raft_consensus.cc:729] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:49.631250 16837 consensus_queue.cc:260] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:49.632015 16837 ts_tablet_manager.cc:1398] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 14:17:49.632746 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-ddf70e508caa4f48a9255be579060d97 received from {username='slave'} at 127.27.220.137:45555
I20240502 14:17:49.632896 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-ddf70e508caa4f48a9255be579060d97 on tablet ddf70e508caa4f48a9255be579060d97 with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:49.670544 16819 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:49.671108 16819 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:49.671494 16819 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 1.017s user 0.064s sys 0.017s
I20240502 14:17:49.672474 16819 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:49.672896 16819 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:49.673189 16819 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:49.674021 16819 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:17:49.675210 15015 raft_consensus.cc:1203] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:49.675230 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-cd382683ec2b4600b6edaced79698b7e received from {username='slave'} at 127.27.220.136:41013
I20240502 14:17:49.675352 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-cd382683ec2b4600b6edaced79698b7e on tablet cd382683ec2b4600b6edaced79698b7e with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:49.678968 17005 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:49.679467 17005 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:49.680043 17005 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.328s user 0.056s sys 0.004s
I20240502 14:17:49.681281 17005 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:49.681982 17005 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:49.682495 17005 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:49.685621 17005 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 14:17:49.686951 12461 tablet_copy_service.cc:342] P eddeff583cf34c8dab2942fb36abbf9e: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-c9ff45109a6d4dc9aba4617819929981 received from {username='slave'} at 127.27.220.136:35863
I20240502 14:17:49.687120 12461 tablet_copy_service.cc:433] P eddeff583cf34c8dab2942fb36abbf9e: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-c9ff45109a6d4dc9aba4617819929981 on tablet c9ff45109a6d4dc9aba4617819929981 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:49.770808 16947 raft_consensus.cc:1050] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:49.771960 16947 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:49.780308 14746 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.781949 16872 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.783856 14883 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.784983 12439 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.785216 12569 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.786528 12702 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.786517 13093 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.787721 16872 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.788152 17129 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.793484 16872 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.794050 16872 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.794586 16872 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.818676 12307 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:49.821166 17126 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:49.824591 17125 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.826009 12700 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.830528 13094 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.831117 12439 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.832451 12161 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:49.835995 14746 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.838788 12307 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:49.847352 12570 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.854507 14883 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.865166 15015 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.865569 14883 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.866268 12834 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.866624 17128 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.867038 17137 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.867529 12439 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.869304 17133 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.869715 17137 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.872063 12963 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.872752 13093 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.873342 17128 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.873914 17128 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.878861 17128 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.879532 12965 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.883241 13093 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.884850 12165 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.887472 12439 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.889376 12552 tablet_service.cc:1514] Processing DeleteTablet for tablet b6bcd8515d234027abfd01ea5dbb3ede with delete_type TABLET_DATA_TOMBSTONED (TS 53479f0fb09c454e9b3cbe7398bf8874 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:59642
I20240502 14:17:49.890064 12154 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b6bcd8515d234027abfd01ea5dbb3ede with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:49.893648 12834 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.899561 15015 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.904120 12307 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.915333 12834 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:49.918174 14883 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.919467 12307 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.920015 17126 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.920363 12700 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.920603 17126 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.921118 17126 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.921438 13094 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.921298 14883 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.922030 14746 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.925590 12439 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:49.926237 16947 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.926798 16947 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.927333 16947 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:49.932327 17131 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.932866 13096 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.934412 12702 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.937813 12155 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4ae8e0c47ce54724899b56aeb1b31336 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:49.938632 12161 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.942482 12307 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.947733 12550 tablet_service.cc:1514] Processing DeleteTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 with delete_type TABLET_DATA_TOMBSTONED (TS 53479f0fb09c454e9b3cbe7398bf8874 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:59642
I20240502 14:17:49.955056 14883 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.958021 14745 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.964576 12441 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:49.978128 17143 tablet_replica.cc:330] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: stopping tablet replica
I20240502 14:17:49.978446 17143 raft_consensus.cc:2229] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:49.978847 17143 raft_consensus.cc:2258] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:49.979843 17143 ts_tablet_manager.cc:1875] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:49.986712 17143 ts_tablet_manager.cc:1888] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:49.986841 17143 log.cc:1196] T b6bcd8515d234027abfd01ea5dbb3ede P 53479f0fb09c454e9b3cbe7398bf8874: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/b6bcd8515d234027abfd01ea5dbb3ede
I20240502 14:17:49.987783 17143 tablet_replica.cc:330] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: stopping tablet replica
I20240502 14:17:49.987839 12152 catalog_manager.cc:4830] TS 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): tablet b6bcd8515d234027abfd01ea5dbb3ede (table rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]) successfully deleted
I20240502 14:17:49.988134 17143 raft_consensus.cc:2229] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:49.988417 17143 raft_consensus.cc:2258] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:49.989436 17143 ts_tablet_manager.cc:1875] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:49.995440 14746 raft_consensus.cc:1203] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:50.007931 17143 ts_tablet_manager.cc:1888] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:50.008124 17143 log.cc:1196] T 4ae8e0c47ce54724899b56aeb1b31336 P 53479f0fb09c454e9b3cbe7398bf8874: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/4ae8e0c47ce54724899b56aeb1b31336
I20240502 14:17:50.008981 12152 catalog_manager.cc:4830] TS 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): tablet 4ae8e0c47ce54724899b56aeb1b31336 (table rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]) successfully deleted
I20240502 14:17:50.022105 12963 consensus_queue.cc:237] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:50.024922 14746 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.025923 13093 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.025986 12307 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.025907 16777 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.026875 12834 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.027027 16777 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.027529 16777 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.028054 14883 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.028681 12439 raft_consensus.cc:1261] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.029347 16777 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.029901 17053 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.030580 17053 consensus_queue.cc:1035] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.035740 14746 raft_consensus.cc:1203] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:50.035373 17052 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:50.038594 12834 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:50.040210 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 29809afc38e54766ba33a4e24311f8f4 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:50.043186 12165 catalog_manager.cc:5484] T 29809afc38e54766ba33a4e24311f8f4 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:50.043991 13093 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:50.044289 14746 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:50.044382 14883 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:50.045861 12307 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:50.046600 12439 raft_consensus.cc:2932] T 29809afc38e54766ba33a4e24311f8f4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:50.058576 12552 tablet_service.cc:1514] Processing DeleteTablet for tablet 29809afc38e54766ba33a4e24311f8f4 with delete_type TABLET_DATA_TOMBSTONED (TS 53479f0fb09c454e9b3cbe7398bf8874 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:59642
I20240502 14:17:50.059103 17143 tablet_replica.cc:330] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: stopping tablet replica
I20240502 14:17:50.059365 17143 raft_consensus.cc:2229] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:50.059674 17143 raft_consensus.cc:2258] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:50.061316 17143 ts_tablet_manager.cc:1875] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:50.069857 12963 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:50.074790 12570 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.074862 12307 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.075377 17143 ts_tablet_manager.cc:1888] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:50.075567 17143 log.cc:1196] T 29809afc38e54766ba33a4e24311f8f4 P 53479f0fb09c454e9b3cbe7398bf8874: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/29809afc38e54766ba33a4e24311f8f4
I20240502 14:17:50.075516 17052 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.076417 12152 catalog_manager.cc:4830] TS 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): tablet 29809afc38e54766ba33a4e24311f8f4 (table rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]) successfully deleted
I20240502 14:17:50.076733 15015 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.077698 14746 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.078155 16645 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.078299 12834 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.078605 17052 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.078886 16996 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.079244 16781 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.082919 16996 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.083496 15015 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.085309 12834 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.089713 12164 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 reported cstate change: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.090867 14745 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.098223 12419 tablet_service.cc:1514] Processing DeleteTablet for tablet 9cd04934b5264d7a9643974c0386cac4 with delete_type TABLET_DATA_TOMBSTONED (TS eddeff583cf34c8dab2942fb36abbf9e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:52872
I20240502 14:17:50.103710 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9cd04934b5264d7a9643974c0386cac4 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:50.104707 12570 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.105672 13093 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.105837 12309 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.106401 16781 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.108060 13096 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.133885 17161 tablet_replica.cc:330] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: stopping tablet replica
I20240502 14:17:50.134258 17161 raft_consensus.cc:2229] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:50.134676 17161 raft_consensus.cc:2258] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:50.135744 17161 ts_tablet_manager.cc:1875] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:50.142590 17161 ts_tablet_manager.cc:1888] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:50.142725 17161 log.cc:1196] T 9cd04934b5264d7a9643974c0386cac4 P eddeff583cf34c8dab2942fb36abbf9e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/9cd04934b5264d7a9643974c0386cac4
I20240502 14:17:50.143672 12153 catalog_manager.cc:4830] TS eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491): tablet 9cd04934b5264d7a9643974c0386cac4 (table rebalance_test_table_5 [id=12cdb9f0a01e41b99ff07ee3111a9291]) successfully deleted
I20240502 14:17:50.152906 17051 raft_consensus.cc:1050] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:50.154543 17051 consensus_queue.cc:237] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:50.159930 14883 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.160707 12439 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.161279 17173 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.161769 15015 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.162137 16680 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.162850 12309 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.162884 16680 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.163404 16353 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.182318 12702 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.182974 13096 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.182749 16680 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.183732 17173 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.184249 17051 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.186632 12834 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.187958 16353 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.195044 12309 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.195065 12834 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.201758 12167 catalog_manager.cc:5484] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.204259 13096 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.210034 14883 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.212154 12702 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.200340 12439 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.215559 15015 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.500702 17173 raft_consensus.cc:1050] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:50.501754 17173 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:50.504710 15015 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.505327 17051 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.507196 14746 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.507357 13096 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.507956 12307 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.507963 12834 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.509424 12439 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.510679 16680 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.511581 12963 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.512563 16680 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.515592 16353 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.518209 16353 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.518784 16353 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.519542 17051 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.525790 16680 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.526582 12963 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.536370 14746 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.543241 12165 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 reported cstate change: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.571382 13096 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.578840 12440 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.580166 15015 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.591603 12307 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.600481 12570 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:50.605013 13093 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.606113 12963 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.605644 15015 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.606846 14746 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.607780 17174 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.608505 12440 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.611119 16353 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.610792 12833 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.611474 17173 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.612628 17051 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.613128 17051 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.616286 17173 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.617961 12833 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.618299 12440 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.618270 14746 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.618757 17237 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.620638 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cd382683ec2b4600b6edaced79698b7e with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:50.622274 12963 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.622556 12164 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:50.624083 15015 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.628546 16567 raft_consensus.cc:1050] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:50.629725 16567 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:50.630831 12833 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.633934 12287 tablet_service.cc:1514] Processing DeleteTablet for tablet cd382683ec2b4600b6edaced79698b7e with delete_type TABLET_DATA_TOMBSTONED (TS 70a425a77fe54cbcb9755f9c540fd367 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:48502
I20240502 14:17:50.633064 13096 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.634959 17241 tablet_replica.cc:330] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: stopping tablet replica
I20240502 14:17:50.635267 17241 raft_consensus.cc:2229] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:50.637709 17241 raft_consensus.cc:2258] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:50.640517 17241 ts_tablet_manager.cc:1875] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:50.647343 17241 ts_tablet_manager.cc:1888] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:50.647495 17241 log.cc:1196] T cd382683ec2b4600b6edaced79698b7e P 70a425a77fe54cbcb9755f9c540fd367: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/cd382683ec2b4600b6edaced79698b7e
I20240502 14:17:50.648217 12154 catalog_manager.cc:4830] TS 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): tablet cd382683ec2b4600b6edaced79698b7e (table rebalance_test_table_0 [id=dc15365c15014c39b8270b3872fb115f]) successfully deleted
I20240502 14:17:50.662875 14746 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.663893 17047 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.665504 14883 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.666072 12570 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.676756 17243 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.677659 17047 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.679420 12833 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.680189 17244 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.680343 12309 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.680977 17242 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.681294 13096 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.682318 17047 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.682897 12702 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:50.683413 17047 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:50.686384 17243 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.686968 12702 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.691920 12165 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.700346 14746 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.702071 14883 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.704896 12309 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.706171 12833 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.713829 12570 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.715974 13096 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.720042 12440 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:50.721671 12569 consensus_queue.cc:237] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:50.727846 12702 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.728663 13094 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.726001 12833 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.729861 12569 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.729874 12702 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.729903 14884 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.730413 12309 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.730700 15015 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.730981 16680 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.731554 16680 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.732076 16680 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.732522 12833 raft_consensus.cc:1261] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.732537 17173 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.732715 14883 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.732817 17051 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.732955 13096 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.733301 17224 consensus_queue.cc:1035] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.733278 17244 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.734103 17244 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.734700 17244 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.735283 14746 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:50.739250 16680 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.739900 17252 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.740515 17047 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.741312 17047 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:50.741391 12309 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.742465 17047 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.744985 12831 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.745704 12702 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.748064 12569 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.749891 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c9ff45109a6d4dc9aba4617819929981 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:50.750308 12700 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.754823 12166 catalog_manager.cc:5484] T ddf70e508caa4f48a9255be579060d97 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.755417 12834 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.760340 12419 tablet_service.cc:1514] Processing DeleteTablet for tablet ddf70e508caa4f48a9255be579060d97 with delete_type TABLET_DATA_TOMBSTONED (TS eddeff583cf34c8dab2942fb36abbf9e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:52872
I20240502 14:17:50.760076 14884 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.761435 13094 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.763247 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ddf70e508caa4f48a9255be579060d97 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:50.764374 14885 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.766012 12164 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.770419 13096 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.772569 15015 raft_consensus.cc:2932] T ddf70e508caa4f48a9255be579060d97 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:50.776314 12287 tablet_service.cc:1514] Processing DeleteTablet for tablet c9ff45109a6d4dc9aba4617819929981 with delete_type TABLET_DATA_TOMBSTONED (TS 70a425a77fe54cbcb9755f9c540fd367 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:48502
I20240502 14:17:50.776736 17241 tablet_replica.cc:330] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: stopping tablet replica
I20240502 14:17:50.776969 17241 raft_consensus.cc:2229] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:50.777455 17241 raft_consensus.cc:2258] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:50.779296 17241 ts_tablet_manager.cc:1875] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:50.787474 17241 ts_tablet_manager.cc:1888] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:50.787202 14746 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:50.787853 17241 log.cc:1196] T c9ff45109a6d4dc9aba4617819929981 P 70a425a77fe54cbcb9755f9c540fd367: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-0/wal/wals/c9ff45109a6d4dc9aba4617819929981
I20240502 14:17:50.788714 12154 catalog_manager.cc:4830] TS 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): tablet c9ff45109a6d4dc9aba4617819929981 (table rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]) successfully deleted
I20240502 14:17:50.795336 17253 tablet_replica.cc:330] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: stopping tablet replica
I20240502 14:17:50.795642 17253 raft_consensus.cc:2229] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:50.796146 17253 raft_consensus.cc:2258] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:50.797415 17253 ts_tablet_manager.cc:1875] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:50.802234 17253 ts_tablet_manager.cc:1888] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 14:17:50.802400 17253 log.cc:1196] T ddf70e508caa4f48a9255be579060d97 P eddeff583cf34c8dab2942fb36abbf9e: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-1/wal/wals/ddf70e508caa4f48a9255be579060d97
I20240502 14:17:50.803205 12153 catalog_manager.cc:4830] TS eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491): tablet ddf70e508caa4f48a9255be579060d97 (table rebalance_test_table_6 [id=c349aa42b5874686acfc20c4512aab6a]) successfully deleted
I20240502 14:17:51.049693 12287 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:51.082679 13074 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:51.102129 14724 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:51.106993 12552 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:17:51.125135 14865 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:51.157920 14996 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:51.179411 12682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:51.188066 12419 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:17:51.225747 12813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:51.233359 12944 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:51.350068 12965 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:51.353473 14745 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:51.354219 16645 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:51.354368 12569 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 14:17:51.354580 12899 consensus_peers.cc:485] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b830809623f543438208dc87476d18c6. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:51.355034 16645 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:51.363020 13094 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:51.363025 15015 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:51.363093 12831 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:51.363636 12309 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:51.363540 17322 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:51.363951 17325 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:51.364312 16777 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:51.364594 16645 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:51.368227 17325 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:51.373215 15015 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:51.380462 12309 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:51.383920 14745 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:51.388522 12167 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:51.383904 12569 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:51.407219 13096 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:51.419389 12831 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:51.946995 17353 ts_tablet_manager.cc:902] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:51.948202 17353 tablet_copy_client.cc:308] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:51.948848 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet b830809623f543438208dc87476d18c6 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:52341)
I20240502 14:17:51.949023 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet b830809623f543438208dc87476d18c6 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:52341: session id = 1ba6a238e6ad4e4a84eff746ff336144-b830809623f543438208dc87476d18c6
I20240502 14:17:51.950976 12985 tablet_copy_source_session.cc:205] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:51.952348 17353 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b830809623f543438208dc87476d18c6. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:51.957842 17353 tablet_copy_client.cc:783] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:51.958091 17353 tablet_copy_client.cc:647] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:51.959571 17353 tablet_copy_client.cc:523] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:51.961786 17353 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:51.998689 17353 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:51.999168 17353 tablet_bootstrap.cc:492] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:51.999506 17353 ts_tablet_manager.cc:1367] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.038s user 0.032s sys 0.004s
I20240502 14:17:52.000284 17353 raft_consensus.cc:348] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:52.000555 17353 raft_consensus.cc:729] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:52.000815 17353 consensus_queue.cc:260] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:52.001703 17353 ts_tablet_manager.cc:1398] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.002s user 0.003s sys 0.001s
I20240502 14:17:52.002931 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-b830809623f543438208dc87476d18c6 received from {username='slave'} at 127.27.220.138:52341
I20240502 14:17:52.003057 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-b830809623f543438208dc87476d18c6 on tablet b830809623f543438208dc87476d18c6 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:52.571048 14883 raft_consensus.cc:1203] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 14:17:53.186756 17377 raft_consensus.cc:1050] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:53.187695 17377 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:53.190848 14883 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:53.191218 14746 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:53.192114 17322 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:53.192824 17322 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:53.193081 12309 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:53.193208 12569 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:53.194068 17397 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:53.194669 15014 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:53.196018 17322 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:53.196512 17322 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:53.204605 13096 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:53.204787 12831 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:53.205446 17377 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:53.206894 17322 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:53.227602 17396 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.233180 12831 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.233991 13096 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.242130 12164 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f reported cstate change: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.243140 15014 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.246145 12309 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.254492 14746 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.256330 14883 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.263257 12569 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } attrs { replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.276067 12963 consensus_queue.cc:237] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } }
I20240502 14:17:53.278515 14883 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:53.278611 12309 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:53.278705 14746 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:53.279215 15014 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:53.279709 13096 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:53.279922 17322 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:53.280133 12831 raft_consensus.cc:1261] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:53.280378 17396 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:53.280614 17397 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:53.280853 17406 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:53.281190 17322 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:53.281999 17398 consensus_queue.cc:1035] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:53.284795 17406 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.286778 15014 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.287693 14883 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.290228 13096 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.290930 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b830809623f543438208dc87476d18c6 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:53.290581 14746 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.292189 12309 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.296634 12166 catalog_manager.cc:5484] T b830809623f543438208dc87476d18c6 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 12 to 13, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.299999 12831 raft_consensus.cc:2932] T b830809623f543438208dc87476d18c6 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } }
I20240502 14:17:53.306162 12552 tablet_service.cc:1514] Processing DeleteTablet for tablet b830809623f543438208dc87476d18c6 with delete_type TABLET_DATA_TOMBSTONED (TS 53479f0fb09c454e9b3cbe7398bf8874 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:59642
I20240502 14:17:53.306991 17414 tablet_replica.cc:330] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: stopping tablet replica
I20240502 14:17:53.307343 17414 raft_consensus.cc:2229] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:53.307916 17414 raft_consensus.cc:2258] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:53.309044 17414 ts_tablet_manager.cc:1875] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:53.313853 17414 ts_tablet_manager.cc:1888] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 14:17:53.313987 17414 log.cc:1196] T b830809623f543438208dc87476d18c6 P 53479f0fb09c454e9b3cbe7398bf8874: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-2/wal/wals/b830809623f543438208dc87476d18c6
I20240502 14:17:53.314764 12152 catalog_manager.cc:4830] TS 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727): tablet b830809623f543438208dc87476d18c6 (table rebalance_test_table_4 [id=913758fcbac24aa7a2e7cb36c5ebf052]) successfully deleted
I20240502 14:17:53.491921 14865 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:53.528283 14724 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:53.538970 12552 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:17:53.579897 13074 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:53.582767 14996 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:53.605104 12287 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:53.654006 12944 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:53.658069 12813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:53.678332 12682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:53.721448 12419 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:17:53.972100 14865 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:53.974483 14724 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:54.002032 12552 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 14:17:54.019251 12682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:54.020974 14996 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 14:17:54.025115 12287 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:54.029771 12813 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 14:17:54.051990 12944 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:54.077306 13074 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 14:17:54.079366 12419 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 14:17:54.244052 12831 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:54.245494 12439 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:54.245951 13096 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } }
I20240502 14:17:54.247781 14883 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.248665 13094 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.248761 12702 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.249001 12307 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.249678 14746 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.250121 17384 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.250515 17544 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.250753 12440 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 14:17:54.252034 12768 consensus_peers.cc:485] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4ae8e0c47ce54724899b56aeb1b31336. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.252851 17384 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.253518 17384 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.254099 17384 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.271414 14746 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 14:17:54.274816 12376 consensus_peers.cc:485] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e -> Peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Couldn't send request to peer af704d4ab2f2496a833c692bd3d25fde. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.274978 14883 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.275775 13094 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.275710 17351 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.276413 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.279909 12963 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.280190 17351 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.280946 17351 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.286586 12831 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.286726 12569 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.288203 12963 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.288205 14883 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.288707 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.289258 14745 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.290202 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.290833 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.294361 17552 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.295192 17552 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 14:17:54.346138 12769 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.136:41465, user_credentials={real_user=slave}} blocked reactor thread for 93496.6us
I20240502 14:17:54.346697 17547 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.348613 17547 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.349014 17351 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.354885 12961 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.362906 12164 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.365131 14885 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.367300 12570 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.383740 12702 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.389835 14743 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.399302 12831 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.399361 12570 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.400051 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.401160 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.409081 12440 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.417500 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.430559 12167 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 10 to 11, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.464227 12834 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.464148 14885 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.467693 12439 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.472707 12309 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.474052 14743 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
W20240502 14:17:54.479430 13031 consensus_peers.cc:485] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): Couldn't send request to peer 70a425a77fe54cbcb9755f9c540fd367. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.490434 16957 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } }
I20240502 14:17:54.500520 12164 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 10 to 11, NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:54.501623 12439 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } }
I20240502 14:17:54.498828 14745 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } }
I20240502 14:17:54.510402 14885 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } }
I20240502 14:17:54.511629 12570 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } }
I20240502 14:17:54.502660 13094 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.536525 12831 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } }
I20240502 14:17:54.542639 12440 consensus_queue.cc:237] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:54.542750 12963 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } }
W20240502 14:17:54.546003 12768 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.137:40043, user_credentials={real_user=slave}} blocked reactor thread for 64360.3us
W20240502 14:17:54.547610 12243 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.134:36329, user_credentials={real_user=slave}} blocked reactor thread for 67742.5us
W20240502 14:17:54.549445 12376 consensus_peers.cc:485] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e -> Peer 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): Couldn't send request to peer 4c04a4a8d8864267be7f67c2abe98f84. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f358e8fda9ca4eeeb1dee5b2011ac029. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.549490 12569 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.550109 17351 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.556175 14883 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.557168 17351 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.572846 12309 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.573516 17351 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.573942 13094 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.574525 12961 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.574805 12831 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.575444 17564 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.576184 17524 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.576843 17524 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.579476 13094 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.602200 17351 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:54.608438 13093 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:54.609805 12831 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:54.611299 14883 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:54.614390 12164 catalog_manager.cc:5484] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:54.617323 12961 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:54.622077 12569 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:54.623406 12309 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } } }
I20240502 14:17:54.644866 12440 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } }
I20240502 14:17:54.653012 14743 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 14:17:54.653550 12375 consensus_peers.cc:485] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e -> Peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329): Couldn't send request to peer d693e089cfa54fbb8df91939410b19db. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.653942 12831 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.653944 12569 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.654541 17564 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.654927 14883 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.654994 17552 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.655433 12702 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.655462 17564 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.656571 17552 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.657119 17552 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.660077 13093 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.660831 17564 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.661295 12961 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:54.673328 17351 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } } }
I20240502 14:17:54.674129 14743 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } } }
I20240502 14:17:54.674705 12702 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } } }
I20240502 14:17:54.677832 12831 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } } }
I20240502 14:17:54.678330 14885 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } } }
I20240502 14:17:54.680924 14745 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.682001 12569 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.683411 17406 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.684165 17406 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.686403 12161 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde reported cstate change: config changed from index 10 to 11, NON_VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) added. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } } }
W20240502 14:17:54.693224 12899 consensus_peers.cc:485] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9cd04934b5264d7a9643974c0386cac4. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.705132 12570 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } } }
I20240502 14:17:54.713713 13093 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER d693e089cfa54fbb8df91939410b19db (127.27.220.134) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } } }
I20240502 14:17:54.719884 17626 ts_tablet_manager.cc:902] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Initiating tablet copy from peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:54.726078 12834 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.728477 17620 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.731451 12309 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.732180 17406 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.733207 15015 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.733999 17406 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.730507 17623 ts_tablet_manager.cc:902] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer c78bd31d8d2142bf9d8e06b5a0074e0f (127.27.220.133:45069)
I20240502 14:17:54.748252 13093 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:54.749001 17620 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:54.773886 17620 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:54.781486 13094 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:54.781893 15015 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:54.790515 12570 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:54.792572 12307 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:54.796432 12164 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:54.809170 17623 tablet_copy_client.cc:308] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.133:45069
I20240502 14:17:54.810170 12854 tablet_copy_service.cc:140] P c78bd31d8d2142bf9d8e06b5a0074e0f: Received BeginTabletCopySession request for tablet 4ae8e0c47ce54724899b56aeb1b31336 from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:36197)
I20240502 14:17:54.810382 12854 tablet_copy_service.cc:161] P c78bd31d8d2142bf9d8e06b5a0074e0f: Beginning new tablet copy session on tablet 4ae8e0c47ce54724899b56aeb1b31336 from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:36197: session id = 95e58ca33b4048e2b000dfbf2b5a0186-4ae8e0c47ce54724899b56aeb1b31336
I20240502 14:17:54.814502 12854 tablet_copy_source_session.cc:205] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:54.816635 17623 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae8e0c47ce54724899b56aeb1b31336. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:54.817147 17626 tablet_copy_client.cc:308] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:34491
I20240502 14:17:54.818361 12461 tablet_copy_service.cc:140] P eddeff583cf34c8dab2942fb36abbf9e: Received BeginTabletCopySession request for tablet 3b3082d181004353bdff4414cd264d7c from peer af704d4ab2f2496a833c692bd3d25fde ({username='slave'} at 127.27.220.132:49687)
I20240502 14:17:54.818539 12461 tablet_copy_service.cc:161] P eddeff583cf34c8dab2942fb36abbf9e: Beginning new tablet copy session on tablet 3b3082d181004353bdff4414cd264d7c from peer af704d4ab2f2496a833c692bd3d25fde at {username='slave'} at 127.27.220.132:49687: session id = af704d4ab2f2496a833c692bd3d25fde-3b3082d181004353bdff4414cd264d7c
I20240502 14:17:54.821329 12461 tablet_copy_source_session.cc:205] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:54.825192 17626 ts_tablet_manager.cc:1875] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 14:17:54.833786 14745 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:54.840215 12831 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:54.844506 17626 ts_tablet_manager.cc:1888] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 14:17:54.844837 17626 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3b3082d181004353bdff4414cd264d7c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:54.858536 17626 tablet_copy_client.cc:783] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:54.858803 17626 tablet_copy_client.cc:647] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:54.861193 17626 tablet_copy_client.cc:523] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:54.862643 13094 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:54.865448 17626 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:54.868281 14883 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
W20240502 14:17:54.868803 13031 consensus_peers.cc:485] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Couldn't send request to peer af704d4ab2f2496a833c692bd3d25fde. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.869375 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:54.869706 12961 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:54.870754 12570 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:54.871320 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:54.871860 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:54.872504 12831 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:54.872117 14745 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:54.874022 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:54.874397 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
W20240502 14:17:54.882022 13031 consensus_peers.cc:485] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): Couldn't send request to peer 70a425a77fe54cbcb9755f9c540fd367. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.882189 12440 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:54.882987 17594 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:54.883389 17527 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.884785 12961 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.886829 12440 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.889266 14883 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.895576 12570 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.900242 12161 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.907119 14743 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.917943 17623 tablet_copy_client.cc:783] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:54.918232 17623 tablet_copy_client.cc:647] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:54.920991 12831 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:54.923066 17623 tablet_copy_client.cc:523] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:54.927055 17623 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:54.953462 12570 consensus_queue.cc:237] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:54.960384 12440 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.960659 12307 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.961122 17237 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.961505 17657 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.963729 17659 ts_tablet_manager.cc:902] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: Initiating tablet copy from peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
W20240502 14:17:54.972971 12506 consensus_peers.cc:485] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a6d444ae07234a30be060fbcc5b63b4f. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:54.974676 12961 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.975234 17237 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.979949 12831 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.980544 17657 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.980988 14745 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.981570 17237 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.983841 13094 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:54.984517 17657 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:54.986586 16680 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:54.987255 14746 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.006090 12307 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.010406 12440 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.011320 17626 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:55.012362 17626 tablet_bootstrap.cc:492] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Bootstrap complete.
I20240502 14:17:55.012732 17626 ts_tablet_manager.cc:1367] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.147s user 0.066s sys 0.014s
I20240502 14:17:55.013940 17626 raft_consensus.cc:348] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:55.013733 12161 catalog_manager.cc:5484] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:55.014331 17626 raft_consensus.cc:729] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: LEARNER
I20240502 14:17:55.014606 17626 consensus_queue.cc:260] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:55.018932 17626 ts_tablet_manager.cc:1398] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.006s user 0.003s sys 0.000s
I20240502 14:17:55.022562 12461 tablet_copy_service.cc:342] P eddeff583cf34c8dab2942fb36abbf9e: Request end of tablet copy session af704d4ab2f2496a833c692bd3d25fde-3b3082d181004353bdff4414cd264d7c received from {username='slave'} at 127.27.220.132:49687
I20240502 14:17:55.022688 12461 tablet_copy_service.cc:433] P eddeff583cf34c8dab2942fb36abbf9e: ending tablet copy session af704d4ab2f2496a833c692bd3d25fde-3b3082d181004353bdff4414cd264d7c on tablet 3b3082d181004353bdff4414cd264d7c with peer af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:55.030843 12833 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.040042 13094 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.040258 12961 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.054764 13093 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
W20240502 14:17:55.061216 13030 consensus_peers.cc:485] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137:40043): Couldn't send request to peer 95e58ca33b4048e2b000dfbf2b5a0186. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: df48755615874edab65577e92249fc78. This is attempt 1: this message will repeat every 5th retry.
W20240502 14:17:55.061738 13031 consensus_peers.cc:485] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Couldn't send request to peer af704d4ab2f2496a833c692bd3d25fde. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:55.063480 14746 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:55.064110 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:55.066267 17659 tablet_copy_client.cc:308] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:34491
I20240502 14:17:55.067463 12461 tablet_copy_service.cc:140] P eddeff583cf34c8dab2942fb36abbf9e: Received BeginTabletCopySession request for tablet f358e8fda9ca4eeeb1dee5b2011ac029 from peer 4c04a4a8d8864267be7f67c2abe98f84 ({username='slave'} at 127.27.220.136:35863)
I20240502 14:17:55.067629 12461 tablet_copy_service.cc:161] P eddeff583cf34c8dab2942fb36abbf9e: Beginning new tablet copy session on tablet f358e8fda9ca4eeeb1dee5b2011ac029 from peer 4c04a4a8d8864267be7f67c2abe98f84 at {username='slave'} at 127.27.220.136:35863: session id = 4c04a4a8d8864267be7f67c2abe98f84-f358e8fda9ca4eeeb1dee5b2011ac029
I20240502 14:17:55.070626 12461 tablet_copy_source_session.cc:205] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:55.072541 17659 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f358e8fda9ca4eeeb1dee5b2011ac029. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:55.073494 14883 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:55.074257 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:55.079007 12963 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:55.079612 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:55.082270 17553 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.084343 12965 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.088794 12831 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:55.089406 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:55.090696 12167 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 12 to 13, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.091055 12831 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.095912 12570 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
W20240502 14:17:55.096897 13031 consensus_peers.cc:485] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf -> Peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613): Couldn't send request to peer 70a425a77fe54cbcb9755f9c540fd367. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:55.096961 17659 tablet_copy_client.cc:783] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:55.097452 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:55.097638 17659 tablet_copy_client.cc:647] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:55.099151 12570 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.105355 17659 tablet_copy_client.cc:523] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:55.107530 12440 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:55.108098 17594 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:55.108762 17659 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap starting.
I20240502 14:17:55.109874 12440 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.116973 17682 ts_tablet_manager.cc:902] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Initiating tablet copy from peer eddeff583cf34c8dab2942fb36abbf9e (127.27.220.130:34491)
I20240502 14:17:55.124392 14883 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.130726 17623 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:55.131204 17623 tablet_bootstrap.cc:492] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:55.131525 17623 ts_tablet_manager.cc:1367] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.205s user 0.049s sys 0.007s
I20240502 14:17:55.131223 14746 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) added. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:55.132329 17623 raft_consensus.cc:348] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:55.132694 17623 raft_consensus.cc:729] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:55.132917 17623 consensus_queue.cc:260] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:55.133769 17623 ts_tablet_manager.cc:1398] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:17:55.137334 15015 raft_consensus.cc:1203] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 14:17:55.139940 12854 tablet_copy_service.cc:342] P c78bd31d8d2142bf9d8e06b5a0074e0f: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-4ae8e0c47ce54724899b56aeb1b31336 received from {username='slave'} at 127.27.220.137:36197
I20240502 14:17:55.140113 12854 tablet_copy_service.cc:433] P c78bd31d8d2142bf9d8e06b5a0074e0f: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-4ae8e0c47ce54724899b56aeb1b31336 on tablet 4ae8e0c47ce54724899b56aeb1b31336 with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:55.156632 12961 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
W20240502 14:17:55.162488 12900 consensus_peers.cc:485] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db -> Peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Couldn't send request to peer af704d4ab2f2496a833c692bd3d25fde. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
W20240502 14:17:55.162695 12899 consensus_peers.cc:485] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9cd04934b5264d7a9643974c0386cac4. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:55.163852 12570 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.164355 14746 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.164402 15015 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.164661 12307 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.164923 17650 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.165195 17628 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.165701 17620 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.166287 17620 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.170616 13093 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.170595 12831 raft_consensus.cc:1261] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer d693e089cfa54fbb8df91939410b19db: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.171228 17406 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.171228 17620 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.172878 12831 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.172983 14745 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.174576 12309 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.178557 17650 consensus_queue.cc:1035] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.181735 13093 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.182775 12161 catalog_manager.cc:5484] T 9cd04934b5264d7a9643974c0386cac4 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New cstate: current_term: 1 leader_uuid: "d693e089cfa54fbb8df91939410b19db" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.187512 15015 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.188141 12570 raft_consensus.cc:2932] T 9cd04934b5264d7a9643974c0386cac4 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.217916 17682 tablet_copy_client.cc:308] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.130:34491
I20240502 14:17:55.218675 12570 consensus_queue.cc:237] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:55.219525 12461 tablet_copy_service.cc:140] P eddeff583cf34c8dab2942fb36abbf9e: Received BeginTabletCopySession request for tablet c9ff45109a6d4dc9aba4617819929981 from peer d693e089cfa54fbb8df91939410b19db ({username='slave'} at 127.27.220.134:51479)
I20240502 14:17:55.219710 12461 tablet_copy_service.cc:161] P eddeff583cf34c8dab2942fb36abbf9e: Beginning new tablet copy session on tablet c9ff45109a6d4dc9aba4617819929981 from peer d693e089cfa54fbb8df91939410b19db at {username='slave'} at 127.27.220.134:51479: session id = d693e089cfa54fbb8df91939410b19db-c9ff45109a6d4dc9aba4617819929981
I20240502 14:17:55.222640 12461 tablet_copy_source_session.cc:205] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:55.224922 17682 ts_tablet_manager.cc:1875] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 14:17:55.233422 17682 ts_tablet_manager.cc:1888] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 14:17:55.233964 17682 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c9ff45109a6d4dc9aba4617819929981. 1 dirs total, 0 dirs full, 0 dirs failed
W20240502 14:17:55.236722 12506 consensus_peers.cc:485] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138:33073): Couldn't send request to peer 1ba6a238e6ad4e4a84eff746ff336144. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 00aebb2064324c3eac8145d1c455622c. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:55.238699 12702 raft_consensus.cc:1203] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 14:17:55.239871 17682 tablet_copy_client.cc:783] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:55.240239 17682 tablet_copy_client.cc:647] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:55.243376 15015 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:55.243428 17682 tablet_copy_client.cc:523] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:55.246145 17670 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:55.249770 17682 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Bootstrap starting.
I20240502 14:17:55.255861 13094 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:55.256520 17670 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:55.275844 12961 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:55.276484 17670 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:55.276603 12831 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:55.277113 12440 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:55.277314 17670 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:55.278555 12307 raft_consensus.cc:1261] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 14:17:55.279248 16680 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:55.279717 16680 consensus_queue.cc:1035] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 14:17:55.291977 12309 consensus_queue.cc:237] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:55.297344 14883 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.297546 15015 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.298485 12965 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.298460 17584 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.299117 13093 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 14:17:55.300652 12244 consensus_peers.cc:485] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 -> Peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Couldn't send request to peer af704d4ab2f2496a833c692bd3d25fde. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:55.301334 12831 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.305207 17584 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.306044 17584 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.308365 17584 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.321866 12965 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:55.320979 17657 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:55.335075 17716 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.342106 12167 catalog_manager.cc:5484] T 00aebb2064324c3eac8145d1c455622c P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:55.346907 12307 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:55.348053 15014 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:55.350523 12831 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:55.376410 17715 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.377542 12961 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.385166 12167 catalog_manager.cc:5484] T b6bcd8515d234027abfd01ea5dbb3ede P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New cstate: current_term: 1 leader_uuid: "70a425a77fe54cbcb9755f9c540fd367" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
W20240502 14:17:55.387845 17697 log.cc:924] Time spent T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: Append to log took a long time: real 0.153s user 0.000s sys 0.000s
I20240502 14:17:55.405562 13094 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:55.407809 12440 raft_consensus.cc:2932] T 00aebb2064324c3eac8145d1c455622c P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } }
I20240502 14:17:55.416198 17623 ts_tablet_manager.cc:902] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:55.416294 13093 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.424619 12439 raft_consensus.cc:1261] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 70a425a77fe54cbcb9755f9c540fd367: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.425981 14885 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.427794 17716 consensus_queue.cc:1035] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.429987 15015 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.437474 12834 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.444583 12439 raft_consensus.cc:2932] T b6bcd8515d234027abfd01ea5dbb3ede P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.470088 17623 tablet_copy_client.cc:308] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:55.470968 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet a6d444ae07234a30be060fbcc5b63b4f from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:45555)
I20240502 14:17:55.471136 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet a6d444ae07234a30be060fbcc5b63b4f from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:45555: session id = 95e58ca33b4048e2b000dfbf2b5a0186-a6d444ae07234a30be060fbcc5b63b4f
I20240502 14:17:55.474274 12592 tablet_copy_source_session.cc:205] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:55.475751 17623 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d444ae07234a30be060fbcc5b63b4f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:55.483042 17623 tablet_copy_client.cc:783] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:55.483415 17737 ts_tablet_manager.cc:902] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:55.484635 17623 tablet_copy_client.cc:647] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:55.486799 17623 tablet_copy_client.cc:523] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:55.491142 17659 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:55.491695 17659 tablet_bootstrap.cc:492] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: Bootstrap complete.
I20240502 14:17:55.492017 17659 ts_tablet_manager.cc:1367] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent bootstrapping tablet: real 0.383s user 0.050s sys 0.002s
I20240502 14:17:55.493019 17659 raft_consensus.cc:348] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:55.493535 17659 raft_consensus.cc:729] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4c04a4a8d8864267be7f67c2abe98f84, State: Initialized, Role: LEARNER
I20240502 14:17:55.503813 17682 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:55.504351 17682 tablet_bootstrap.cc:492] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Bootstrap complete.
I20240502 14:17:55.504667 17682 ts_tablet_manager.cc:1367] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Time spent bootstrapping tablet: real 0.255s user 0.062s sys 0.020s
I20240502 14:17:55.505638 17682 raft_consensus.cc:348] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } }
I20240502 14:17:55.506044 17682 raft_consensus.cc:729] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d693e089cfa54fbb8df91939410b19db, State: Initialized, Role: LEARNER
I20240502 14:17:55.506263 17682 consensus_queue.cc:260] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: NON_VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: true } }
I20240502 14:17:55.493824 17659 consensus_queue.cc:260] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: NON_VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: true } }
I20240502 14:17:55.507869 17659 ts_tablet_manager.cc:1398] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84: Time spent starting tablet: real 0.016s user 0.005s sys 0.000s
I20240502 14:17:55.509595 12461 tablet_copy_service.cc:342] P eddeff583cf34c8dab2942fb36abbf9e: Request end of tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-f358e8fda9ca4eeeb1dee5b2011ac029 received from {username='slave'} at 127.27.220.136:35863
I20240502 14:17:55.509732 12461 tablet_copy_service.cc:433] P eddeff583cf34c8dab2942fb36abbf9e: ending tablet copy session 4c04a4a8d8864267be7f67c2abe98f84-f358e8fda9ca4eeeb1dee5b2011ac029 on tablet f358e8fda9ca4eeeb1dee5b2011ac029 with peer 4c04a4a8d8864267be7f67c2abe98f84
I20240502 14:17:55.515699 17623 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:55.517004 17682 ts_tablet_manager.cc:1398] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db: Time spent starting tablet: real 0.012s user 0.005s sys 0.000s
I20240502 14:17:55.517874 12461 tablet_copy_service.cc:342] P eddeff583cf34c8dab2942fb36abbf9e: Request end of tablet copy session d693e089cfa54fbb8df91939410b19db-c9ff45109a6d4dc9aba4617819929981 received from {username='slave'} at 127.27.220.134:51479
I20240502 14:17:55.517993 12461 tablet_copy_service.cc:433] P eddeff583cf34c8dab2942fb36abbf9e: ending tablet copy session d693e089cfa54fbb8df91939410b19db-c9ff45109a6d4dc9aba4617819929981 on tablet c9ff45109a6d4dc9aba4617819929981 with peer d693e089cfa54fbb8df91939410b19db
I20240502 14:17:55.531396 17737 tablet_copy_client.cc:308] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:55.532341 13115 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet df48755615874edab65577e92249fc78 from peer 70a425a77fe54cbcb9755f9c540fd367 ({username='slave'} at 127.27.220.129:42505)
I20240502 14:17:55.532512 13115 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet df48755615874edab65577e92249fc78 from peer 70a425a77fe54cbcb9755f9c540fd367 at {username='slave'} at 127.27.220.129:42505: session id = 70a425a77fe54cbcb9755f9c540fd367-df48755615874edab65577e92249fc78
I20240502 14:17:55.535063 13115 tablet_copy_source_session.cc:205] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:55.537161 17737 ts_tablet_manager.cc:1875] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 14:17:55.544435 17737 ts_tablet_manager.cc:1888] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.9
I20240502 14:17:55.544767 17737 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:55.555095 12570 consensus_queue.cc:237] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:55.559967 14746 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.561121 12834 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.561239 13094 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.562024 15015 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.562705 12961 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.564821 16680 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.565495 16680 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.005s
I20240502 14:17:55.571427 16680 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 14:17:55.574018 12507 consensus_peers.cc:485] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 -> Peer af704d4ab2f2496a833c692bd3d25fde (127.27.220.132:45069): Couldn't send request to peer af704d4ab2f2496a833c692bd3d25fde. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 14:17:55.574503 17237 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.575843 17670 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.579525 17737 tablet_copy_client.cc:783] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:55.579905 17737 tablet_copy_client.cc:647] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:55.589982 17755 ts_tablet_manager.cc:902] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:55.593060 12440 raft_consensus.cc:1261] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 14:17:55.592507 17749 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.593341 17737 tablet_copy_client.cc:523] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:55.593693 16680 consensus_queue.cc:1035] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 14:17:55.602967 12834 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.603209 14746 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.604518 17737 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap starting.
I20240502 14:17:55.606047 13094 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.611120 12961 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.614244 12165 catalog_manager.cc:5484] T cd382683ec2b4600b6edaced79698b7e P 4c04a4a8d8864267be7f67c2abe98f84 reported cstate change: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.620141 15015 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.631708 12440 raft_consensus.cc:2932] T cd382683ec2b4600b6edaced79698b7e P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } }
I20240502 14:17:55.661110 12961 raft_consensus.cc:1203] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 14:17:55.671150 17767 ts_tablet_manager.cc:902] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:55.678619 17772 ts_tablet_manager.cc:902] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Initiating tablet copy from peer 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203)
I20240502 14:17:55.688181 17772 tablet_copy_client.cc:308] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:55.689039 13115 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet df48755615874edab65577e92249fc78 from peer af704d4ab2f2496a833c692bd3d25fde ({username='slave'} at 127.27.220.132:48733)
I20240502 14:17:55.689240 13115 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet df48755615874edab65577e92249fc78 from peer af704d4ab2f2496a833c692bd3d25fde at {username='slave'} at 127.27.220.132:48733: session id = af704d4ab2f2496a833c692bd3d25fde-df48755615874edab65577e92249fc78
I20240502 14:17:55.692329 13115 tablet_copy_source_session.cc:205] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:55.694772 17772 ts_tablet_manager.cc:1875] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 14:17:55.698616 17755 tablet_copy_client.cc:308] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:55.699782 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet 9cd04934b5264d7a9643974c0386cac4 from peer af704d4ab2f2496a833c692bd3d25fde ({username='slave'} at 127.27.220.132:59755)
I20240502 14:17:55.699949 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet 9cd04934b5264d7a9643974c0386cac4 from peer af704d4ab2f2496a833c692bd3d25fde at {username='slave'} at 127.27.220.132:59755: session id = af704d4ab2f2496a833c692bd3d25fde-9cd04934b5264d7a9643974c0386cac4
I20240502 14:17:55.706399 17772 ts_tablet_manager.cc:1888] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 14:17:55.708638 12985 tablet_copy_source_session.cc:205] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:55.710065 17772 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:55.710253 17755 ts_tablet_manager.cc:1875] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 14:17:55.714246 17701 raft_consensus.cc:1050] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e: attempting to promote NON_VOTER af704d4ab2f2496a833c692bd3d25fde to VOTER
I20240502 14:17:55.715409 17701 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } }
I20240502 14:17:55.719096 17755 ts_tablet_manager.cc:1888] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 14:17:55.719441 17755 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:55.721205 17767 tablet_copy_client.cc:308] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.135:44203
I20240502 14:17:55.721714 17772 tablet_copy_client.cc:783] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:55.722051 13115 tablet_copy_service.cc:140] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Received BeginTabletCopySession request for tablet df48755615874edab65577e92249fc78 from peer 95e58ca33b4048e2b000dfbf2b5a0186 ({username='slave'} at 127.27.220.137:46251)
I20240502 14:17:55.722216 13115 tablet_copy_service.cc:161] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Beginning new tablet copy session on tablet df48755615874edab65577e92249fc78 from peer 95e58ca33b4048e2b000dfbf2b5a0186 at {username='slave'} at 127.27.220.137:46251: session id = 95e58ca33b4048e2b000dfbf2b5a0186-df48755615874edab65577e92249fc78
I20240502 14:17:55.725788 13115 tablet_copy_source_session.cc:205] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:55.726488 17755 tablet_copy_client.cc:783] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:55.727610 17767 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet df48755615874edab65577e92249fc78. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:55.731674 17772 tablet_copy_client.cc:647] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:55.731719 17755 tablet_copy_client.cc:647] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:55.733107 17587 raft_consensus.cc:1050] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:55.737726 17587 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:55.743716 17737 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap replayed 1/1 log segments. Stats: ops{read=13 overwritten=0 applied=13 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:55.744361 17737 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Bootstrap complete.
I20240502 14:17:55.744388 17755 tablet_copy_client.cc:523] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:55.744735 17737 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Time spent bootstrapping tablet: real 0.140s user 0.054s sys 0.008s
I20240502 14:17:55.747330 17767 tablet_copy_client.cc:783] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:55.747640 17767 tablet_copy_client.cc:647] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:55.749486 17767 tablet_copy_client.cc:523] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:55.749502 12702 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.750026 14746 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.750447 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.755113 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.755957 17767 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap starting.
I20240502 14:17:55.757658 17755 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:55.757858 17737 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:55.758399 17737 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 70a425a77fe54cbcb9755f9c540fd367, State: Initialized, Role: LEARNER
I20240502 14:17:55.758754 17737 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 13, Last appended: 1.13, Last appended by leader: 13, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:55.759855 17737 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367: Time spent starting tablet: real 0.015s user 0.006s sys 0.000s
I20240502 14:17:55.761898 13115 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 70a425a77fe54cbcb9755f9c540fd367-df48755615874edab65577e92249fc78 received from {username='slave'} at 127.27.220.129:42505
I20240502 14:17:55.762077 13115 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 70a425a77fe54cbcb9755f9c540fd367-df48755615874edab65577e92249fc78 on tablet df48755615874edab65577e92249fc78 with peer 70a425a77fe54cbcb9755f9c540fd367
I20240502 14:17:55.762619 17772 tablet_copy_client.cc:523] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:55.768489 14883 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.768944 13094 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.769632 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.769981 17623 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:55.770252 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.770483 17623 tablet_bootstrap.cc:492] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:55.770825 17623 ts_tablet_manager.cc:1367] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.255s user 0.051s sys 0.016s
I20240502 14:17:55.775796 14884 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.776784 17772 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:55.776815 17766 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.777029 14746 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.777969 13093 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.779057 17623 raft_consensus.cc:348] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:55.779507 17623 raft_consensus.cc:729] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:55.779794 17623 consensus_queue.cc:260] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:55.789438 17623 ts_tablet_manager.cc:1398] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.018s user 0.004s sys 0.000s
I20240502 14:17:55.790165 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-a6d444ae07234a30be060fbcc5b63b4f received from {username='slave'} at 127.27.220.137:45555
I20240502 14:17:55.790341 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-a6d444ae07234a30be060fbcc5b63b4f on tablet a6d444ae07234a30be060fbcc5b63b4f with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:55.797772 12961 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.798429 17351 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.798624 12570 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.799374 17351 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.808624 17798 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.809244 17798 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.812732 17800 ts_tablet_manager.cc:902] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer d693e089cfa54fbb8df91939410b19db (127.27.220.134:36329)
I20240502 14:17:55.815482 15015 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.816318 12440 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.819350 12831 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.819880 17701 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.821039 17587 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.821574 17587 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
W20240502 14:17:55.853868 12769 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.27.220.135:44203, user_credentials={real_user=slave}} blocked reactor thread for 72907.9us
I20240502 14:17:55.855268 12700 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.857239 12309 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:55.868332 17797 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.870666 17798 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:55.874622 17789 ts_tablet_manager.cc:902] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:55.881690 17351 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.887020 17811 ts_tablet_manager.cc:902] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Initiating tablet copy from peer 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129:46613)
I20240502 14:17:55.890065 12961 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.895807 12702 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.895942 13094 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.896955 12161 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.899991 14883 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.895427 17797 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:55.906538 14746 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:55.905390 14745 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.910778 12831 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.909081 12309 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:55.930382 12700 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:55.933136 12570 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:55.938846 17789 tablet_copy_client.cc:308] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:55.939086 12165 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:55.940145 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet 00aebb2064324c3eac8145d1c455622c from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:39309)
I20240502 14:17:55.940333 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet 00aebb2064324c3eac8145d1c455622c from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:39309: session id = 1ba6a238e6ad4e4a84eff746ff336144-00aebb2064324c3eac8145d1c455622c
I20240502 14:17:55.943132 12592 tablet_copy_source_session.cc:205] T 00aebb2064324c3eac8145d1c455622c P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:55.949172 17789 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00aebb2064324c3eac8145d1c455622c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:55.956451 13093 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:55.960978 12440 consensus_queue.cc:237] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } }
I20240502 14:17:55.963121 14884 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:55.970036 12702 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:55.970569 15015 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:55.971858 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:55.980125 17789 tablet_copy_client.cc:783] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:55.980401 17789 tablet_copy_client.cc:647] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:55.982403 17789 tablet_copy_client.cc:523] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:55.988314 15015 raft_consensus.cc:1203] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:55.994717 12831 consensus_queue.cc:237] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:55.994490 12439 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.001081 12702 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.001755 17587 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.001971 13093 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.002686 15015 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.003471 17796 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.006106 17796 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.019157 14885 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.019924 17798 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.021966 12439 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.023113 14885 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.025638 17587 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.027155 17767 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap replayed 1/1 log segments. Stats: ops{read=13 overwritten=0 applied=13 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:56.027626 17767 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: Bootstrap complete.
I20240502 14:17:56.027943 17767 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent bootstrapping tablet: real 0.272s user 0.058s sys 0.012s
I20240502 14:17:56.028774 17767 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:56.029227 17767 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 95e58ca33b4048e2b000dfbf2b5a0186, State: Initialized, Role: LEARNER
I20240502 14:17:56.029628 12439 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.031594 17587 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.029464 17767 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 13, Last appended: 1.13, Last appended by leader: 13, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:56.033138 17767 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 14:17:56.034482 13115 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-df48755615874edab65577e92249fc78 received from {username='slave'} at 127.27.220.137:46251
I20240502 14:17:56.034657 13115 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session 95e58ca33b4048e2b000dfbf2b5a0186-df48755615874edab65577e92249fc78 on tablet df48755615874edab65577e92249fc78 with peer 95e58ca33b4048e2b000dfbf2b5a0186
I20240502 14:17:56.038868 12570 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.039358 12961 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.041491 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.042074 12834 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.042814 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.044222 17524 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.048712 12155 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4ae8e0c47ce54724899b56aeb1b31336 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:56.049017 17800 tablet_copy_client.cc:308] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.134:36329
I20240502 14:17:56.049705 12985 tablet_copy_service.cc:140] P d693e089cfa54fbb8df91939410b19db: Received BeginTabletCopySession request for tablet 9cd04934b5264d7a9643974c0386cac4 from peer 1ba6a238e6ad4e4a84eff746ff336144 ({username='slave'} at 127.27.220.138:52341)
I20240502 14:17:56.049876 12985 tablet_copy_service.cc:161] P d693e089cfa54fbb8df91939410b19db: Beginning new tablet copy session on tablet 9cd04934b5264d7a9643974c0386cac4 from peer 1ba6a238e6ad4e4a84eff746ff336144 at {username='slave'} at 127.27.220.138:52341: session id = 1ba6a238e6ad4e4a84eff746ff336144-9cd04934b5264d7a9643974c0386cac4
I20240502 14:17:56.049896 12834 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:56.050375 12570 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:56.051653 17524 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:56.052774 12985 tablet_copy_source_session.cc:205] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:56.056329 12702 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.058236 17800 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9cd04934b5264d7a9643974c0386cac4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:56.065145 14746 raft_consensus.cc:1203] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:56.064071 12165 catalog_manager.cc:5484] T 4ae8e0c47ce54724899b56aeb1b31336 P 1ba6a238e6ad4e4a84eff746ff336144 reported cstate change: config changed from index 12 to 13, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New cstate: current_term: 1 leader_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.067209 12309 raft_consensus.cc:1203] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.12->[1.13-1.13] Dedup: 1.13->[]
I20240502 14:17:56.075189 14884 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.076617 17821 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.078894 17836 ts_tablet_manager.cc:902] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Initiating tablet copy from peer 53479f0fb09c454e9b3cbe7398bf8874 (127.27.220.131:43727)
I20240502 14:17:56.085232 15015 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.096230 14743 raft_consensus.cc:1261] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.096390 17755 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:56.097889 17821 consensus_queue.cc:1035] T 3b3082d181004353bdff4414cd264d7c P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.098127 17755 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap complete.
I20240502 14:17:56.099884 17755 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.351s user 0.064s sys 0.003s
I20240502 14:17:56.100641 17789 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:56.100728 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3b3082d181004353bdff4414cd264d7c with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:56.100822 17755 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.101250 17755 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: LEARNER
I20240502 14:17:56.101372 12307 raft_consensus.cc:1261] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer c78bd31d8d2142bf9d8e06b5a0074e0f: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.101562 17755 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 1.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.102502 12700 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:56.102507 13094 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.104281 17834 consensus_queue.cc:1035] T 4ae8e0c47ce54724899b56aeb1b31336 P c78bd31d8d2142bf9d8e06b5a0074e0f [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.106704 17811 tablet_copy_client.cc:308] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.129:46613
I20240502 14:17:56.106290 12307 raft_consensus.cc:2932] T 4ae8e0c47ce54724899b56aeb1b31336 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.107628 12329 tablet_copy_service.cc:140] P 70a425a77fe54cbcb9755f9c540fd367: Received BeginTabletCopySession request for tablet b6bcd8515d234027abfd01ea5dbb3ede from peer af704d4ab2f2496a833c692bd3d25fde ({username='slave'} at 127.27.220.132:42339)
I20240502 14:17:56.107856 12329 tablet_copy_service.cc:161] P 70a425a77fe54cbcb9755f9c540fd367: Beginning new tablet copy session on tablet b6bcd8515d234027abfd01ea5dbb3ede from peer af704d4ab2f2496a833c692bd3d25fde at {username='slave'} at 127.27.220.132:42339: session id = af704d4ab2f2496a833c692bd3d25fde-b6bcd8515d234027abfd01ea5dbb3ede
I20240502 14:17:56.108927 14745 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:56.111063 12329 tablet_copy_source_session.cc:205] T b6bcd8515d234027abfd01ea5dbb3ede P 70a425a77fe54cbcb9755f9c540fd367: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:56.112517 17811 ts_tablet_manager.cc:1875] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 14:17:56.116835 12161 catalog_manager.cc:5484] T 3b3082d181004353bdff4414cd264d7c P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:56.118053 17755 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.018s user 0.002s sys 0.000s
I20240502 14:17:56.119244 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session af704d4ab2f2496a833c692bd3d25fde-9cd04934b5264d7a9643974c0386cac4 received from {username='slave'} at 127.27.220.132:59755
I20240502 14:17:56.119395 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session af704d4ab2f2496a833c692bd3d25fde-9cd04934b5264d7a9643974c0386cac4 on tablet 9cd04934b5264d7a9643974c0386cac4 with peer af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:56.120136 12961 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:56.123805 14883 raft_consensus.cc:2932] T 3b3082d181004353bdff4414cd264d7c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } }
I20240502 14:17:56.129411 17811 ts_tablet_manager.cc:1888] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 14:17:56.129806 17811 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b6bcd8515d234027abfd01ea5dbb3ede. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:56.142063 17811 tablet_copy_client.cc:783] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:56.142518 17811 tablet_copy_client.cc:647] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:56.144390 13074 tablet_service.cc:1514] Processing DeleteTablet for tablet 3b3082d181004353bdff4414cd264d7c with delete_type TABLET_DATA_TOMBSTONED (TS 84d1fcd2d4cf472db7959f63a7f0ddaf not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:46746
I20240502 14:17:56.147305 17836 tablet_copy_client.cc:308] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Beginning tablet copy session from remote peer at address 127.27.220.131:43727
I20240502 14:17:56.148092 12592 tablet_copy_service.cc:140] P 53479f0fb09c454e9b3cbe7398bf8874: Received BeginTabletCopySession request for tablet cd382683ec2b4600b6edaced79698b7e from peer af704d4ab2f2496a833c692bd3d25fde ({username='slave'} at 127.27.220.132:56877)
I20240502 14:17:56.148331 12592 tablet_copy_service.cc:161] P 53479f0fb09c454e9b3cbe7398bf8874: Beginning new tablet copy session on tablet cd382683ec2b4600b6edaced79698b7e from peer af704d4ab2f2496a833c692bd3d25fde at {username='slave'} at 127.27.220.132:56877: session id = af704d4ab2f2496a833c692bd3d25fde-cd382683ec2b4600b6edaced79698b7e
I20240502 14:17:56.150017 17811 tablet_copy_client.cc:523] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:56.151674 12592 tablet_copy_source_session.cc:205] T cd382683ec2b4600b6edaced79698b7e P 53479f0fb09c454e9b3cbe7398bf8874: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 14:17:56.152763 17836 ts_tablet_manager.cc:1875] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 14:17:56.159945 17811 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:56.166160 17849 tablet_replica.cc:330] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: stopping tablet replica
I20240502 14:17:56.166697 17849 raft_consensus.cc:2229] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:56.167104 17849 raft_consensus.cc:2258] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:56.171443 17800 tablet_copy_client.cc:783] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:56.171803 17800 tablet_copy_client.cc:647] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:56.179440 17836 ts_tablet_manager.cc:1888] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 14:17:56.183151 17836 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cd382683ec2b4600b6edaced79698b7e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 14:17:56.185200 17800 tablet_copy_client.cc:523] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:56.191985 17800 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap starting.
I20240502 14:17:56.196339 14724 tablet_service.cc:1514] Processing DeleteTablet for tablet 4ae8e0c47ce54724899b56aeb1b31336 with delete_type TABLET_DATA_TOMBSTONED (TS 4c04a4a8d8864267be7f67c2abe98f84 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:55362
I20240502 14:17:56.208782 15015 raft_consensus.cc:1203] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.12->[1.13-1.13] Dedup: 1.13->[]
I20240502 14:17:56.213315 17836 tablet_copy_client.cc:783] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 0 data blocks...
I20240502 14:17:56.213658 17836 tablet_copy_client.cc:647] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Starting download of 1 WAL segments...
I20240502 14:17:56.217473 17836 tablet_copy_client.cc:523] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 14:17:56.218274 17854 tablet_replica.cc:330] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: stopping tablet replica
I20240502 14:17:56.219058 17854 raft_consensus.cc:2229] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:56.222175 17849 ts_tablet_manager.cc:1875] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:56.224679 17854 raft_consensus.cc:2258] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:56.229238 17836 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Bootstrap starting.
I20240502 14:17:56.231775 17854 ts_tablet_manager.cc:1875] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:56.249827 17849 ts_tablet_manager.cc:1888] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 14:17:56.250073 17849 log.cc:1196] T 3b3082d181004353bdff4414cd264d7c P 84d1fcd2d4cf472db7959f63a7f0ddaf: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/wals/3b3082d181004353bdff4414cd264d7c
I20240502 14:17:56.251080 17854 ts_tablet_manager.cc:1888] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 14:17:56.251367 17854 log.cc:1196] T 4ae8e0c47ce54724899b56aeb1b31336 P 4c04a4a8d8864267be7f67c2abe98f84: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-7/wal/wals/4ae8e0c47ce54724899b56aeb1b31336
I20240502 14:17:56.251366 12154 catalog_manager.cc:4830] TS 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): tablet 3b3082d181004353bdff4414cd264d7c (table rebalance_test_table_2 [id=4a4559d5f7734ac2856118a864770cb5]) successfully deleted
I20240502 14:17:56.252415 12154 catalog_manager.cc:4830] TS 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136:41465): tablet 4ae8e0c47ce54724899b56aeb1b31336 (table rebalance_test_table_1 [id=8d0945dae96041c5bd73b675e5da6c37]) successfully deleted
I20240502 14:17:56.283715 17846 raft_consensus.cc:1050] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e: attempting to promote NON_VOTER d693e089cfa54fbb8df91939410b19db to VOTER
I20240502 14:17:56.285053 17846 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } }
I20240502 14:17:56.298661 12961 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 LEARNER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:56.299474 17524 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:56.300427 17800 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:56.300989 17800 tablet_bootstrap.cc:492] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:56.301416 17800 ts_tablet_manager.cc:1367] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.110s user 0.054s sys 0.004s
I20240502 14:17:56.302294 17800 raft_consensus.cc:348] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.302657 17800 raft_consensus.cc:729] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:56.302892 17800 consensus_queue.cc:260] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 1.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.303671 17800 ts_tablet_manager.cc:1398] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 14:17:56.304607 12985 tablet_copy_service.cc:342] P d693e089cfa54fbb8df91939410b19db: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-9cd04934b5264d7a9643974c0386cac4 received from {username='slave'} at 127.27.220.138:52341
I20240502 14:17:56.304733 12985 tablet_copy_service.cc:433] P d693e089cfa54fbb8df91939410b19db: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-9cd04934b5264d7a9643974c0386cac4 on tablet 9cd04934b5264d7a9643974c0386cac4 with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:56.309784 14883 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:56.309815 14745 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:56.310456 12570 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:56.311880 12831 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:56.313472 17789 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:56.313611 13094 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:56.313939 17789 tablet_bootstrap.cc:492] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: Bootstrap complete.
I20240502 14:17:56.314237 17789 ts_tablet_manager.cc:1367] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: Time spent bootstrapping tablet: real 0.214s user 0.063s sys 0.035s
I20240502 14:17:56.314211 12700 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 14:17:56.315022 17789 raft_consensus.cc:348] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:56.315392 17789 raft_consensus.cc:729] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1ba6a238e6ad4e4a84eff746ff336144, State: Initialized, Role: LEARNER
I20240502 14:17:56.316174 17351 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:56.316753 17351 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:56.315696 17789 consensus_queue.cc:260] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: NON_VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: true } }
I20240502 14:17:56.317353 17351 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:56.318239 17789 ts_tablet_manager.cc:1398] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240502 14:17:56.318627 17772 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap replayed 1/1 log segments. Stats: ops{read=13 overwritten=0 applied=13 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:56.318990 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-00aebb2064324c3eac8145d1c455622c received from {username='slave'} at 127.27.220.138:39309
I20240502 14:17:56.319079 17772 tablet_bootstrap.cc:492] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Bootstrap complete.
I20240502 14:17:56.319130 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session 1ba6a238e6ad4e4a84eff746ff336144-00aebb2064324c3eac8145d1c455622c on tablet 00aebb2064324c3eac8145d1c455622c with peer 1ba6a238e6ad4e4a84eff746ff336144
I20240502 14:17:56.319770 17835 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:56.320104 17524 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:56.320598 17524 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 14:17:56.334645 17772 ts_tablet_manager.cc:1367] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.558s user 0.071s sys 0.009s
I20240502 14:17:56.340123 17772 raft_consensus.cc:348] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:56.340642 17772 raft_consensus.cc:729] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: LEARNER
I20240502 14:17:56.340934 17772 consensus_queue.cc:260] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 13, Last appended: 1.13, Last appended by leader: 13, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: NON_VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: true } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:56.341888 17772 ts_tablet_manager.cc:1398] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 14:17:56.348335 14883 raft_consensus.cc:1203] T 9cd04934b5264d7a9643974c0386cac4 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.11->[1.12-1.12] Dedup: 1.12->[]
I20240502 14:17:56.353220 13115 tablet_copy_service.cc:342] P 84d1fcd2d4cf472db7959f63a7f0ddaf: Request end of tablet copy session af704d4ab2f2496a833c692bd3d25fde-df48755615874edab65577e92249fc78 received from {username='slave'} at 127.27.220.132:48733
I20240502 14:17:56.353415 13115 tablet_copy_service.cc:433] P 84d1fcd2d4cf472db7959f63a7f0ddaf: ending tablet copy session af704d4ab2f2496a833c692bd3d25fde-df48755615874edab65577e92249fc78 on tablet df48755615874edab65577e92249fc78 with peer af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:56.378159 17351 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.382153 14745 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.388815 12700 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.388926 12165 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 reported cstate change: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.393081 12702 raft_consensus.cc:1203] T 9cd04934b5264d7a9643974c0386cac4 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Deduplicated request from leader. Original: 1.11->[1.12-1.12] Dedup: 1.12->[]
I20240502 14:17:56.402122 13094 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.406229 14883 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.406894 12965 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.425738 12570 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.426564 12831 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, d693e089cfa54fbb8df91939410b19db (127.27.220.134) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.430717 12439 consensus_queue.cc:237] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } }
I20240502 14:17:56.435314 16680 raft_consensus.cc:1050] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:56.435534 14745 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.436522 14883 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.436589 16680 consensus_queue.cc:237] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:56.437724 17524 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.437999 12965 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.439296 12569 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.440809 17846 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.441197 17835 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.445457 17835 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.445974 12569 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.447402 12700 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.448639 17701 raft_consensus.cc:1050] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e: attempting to promote NON_VOTER 4c04a4a8d8864267be7f67c2abe98f84 to VOTER
I20240502 14:17:56.448678 17821 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.001s
I20240502 14:17:56.450554 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c9ff45109a6d4dc9aba4617819929981 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:56.453066 12834 raft_consensus.cc:1261] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 14:17:56.454106 12965 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.454568 14883 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.460279 12165 catalog_manager.cc:5484] T c9ff45109a6d4dc9aba4617819929981 P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.465807 13074 tablet_service.cc:1514] Processing DeleteTablet for tablet c9ff45109a6d4dc9aba4617819929981 with delete_type TABLET_DATA_TOMBSTONED (TS 84d1fcd2d4cf472db7959f63a7f0ddaf not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:46746
I20240502 14:17:56.466346 17849 tablet_replica.cc:330] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: stopping tablet replica
I20240502 14:17:56.466538 17849 raft_consensus.cc:2229] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 14:17:56.466800 17849 raft_consensus.cc:2258] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 14:17:56.468427 17849 ts_tablet_manager.cc:1875] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 14:17:56.469355 14745 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.474347 17351 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.007s
I20240502 14:17:56.474470 17701 consensus_queue.cc:237] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:56.478746 12700 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.479303 17846 consensus_queue.cc:1035] T c9ff45109a6d4dc9aba4617819929981 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 14:17:56.481453 12834 raft_consensus.cc:2932] T c9ff45109a6d4dc9aba4617819929981 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { promote: false } } }
I20240502 14:17:56.485484 17527 raft_consensus.cc:1050] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 70a425a77fe54cbcb9755f9c540fd367 to VOTER
I20240502 14:17:56.486954 17527 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 1.13, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }
I20240502 14:17:56.493063 17849 ts_tablet_manager.cc:1888] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 14:17:56.493356 12309 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.494019 17657 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.494418 17849 log.cc:1196] T c9ff45109a6d4dc9aba4617819929981 P 84d1fcd2d4cf472db7959f63a7f0ddaf: Deleting WAL directory at /tmp/dist-test-taskhB7nWc/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714659255136012-28530-0/raft_consensus-itest-cluster/ts-6/wal/wals/c9ff45109a6d4dc9aba4617819929981
I20240502 14:17:56.497686 12154 catalog_manager.cc:4830] TS 84d1fcd2d4cf472db7959f63a7f0ddaf (127.27.220.135:44203): tablet c9ff45109a6d4dc9aba4617819929981 (table rebalance_test_table_3 [id=75493d67ecb34d899058cbff9d4e602d]) successfully deleted
I20240502 14:17:56.500015 12439 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.501075 17759 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.502276 15015 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.503002 17759 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.504153 12570 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.504765 17846 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.504796 14745 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.506888 17749 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.510386 12965 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.510872 12831 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.510999 17701 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.511513 17846 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.512087 14883 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.513093 17701 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.527539 13094 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.527927 12961 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.529600 17759 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.530220 17759 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.550777 12834 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.551561 17657 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.558229 17811 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:56.558825 17811 tablet_bootstrap.cc:492] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Bootstrap complete.
I20240502 14:17:56.559216 17811 ts_tablet_manager.cc:1367] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.399s user 0.067s sys 0.001s
I20240502 14:17:56.560243 17811 raft_consensus.cc:348] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.560747 17811 raft_consensus.cc:729] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: LEARNER
I20240502 14:17:56.561009 17811 consensus_queue.cc:260] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.561802 17811 ts_tablet_manager.cc:1398] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 14:17:56.565624 14743 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 LEARNER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.566028 12307 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.566429 13093 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 14:17:56.568284 17701 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.569156 17701 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.569975 17846 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 14:17:56.572741 12329 tablet_copy_service.cc:342] P 70a425a77fe54cbcb9755f9c540fd367: Request end of tablet copy session af704d4ab2f2496a833c692bd3d25fde-b6bcd8515d234027abfd01ea5dbb3ede received from {username='slave'} at 127.27.220.132:42339
I20240502 14:17:56.572906 12329 tablet_copy_service.cc:433] P 70a425a77fe54cbcb9755f9c540fd367: ending tablet copy session af704d4ab2f2496a833c692bd3d25fde-b6bcd8515d234027abfd01ea5dbb3ede on tablet b6bcd8515d234027abfd01ea5dbb3ede with peer af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:56.579447 17836 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 14:17:56.580010 17836 tablet_bootstrap.cc:492] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Bootstrap complete.
I20240502 14:17:56.580322 17836 ts_tablet_manager.cc:1367] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Time spent bootstrapping tablet: real 0.351s user 0.059s sys 0.001s
I20240502 14:17:56.581142 17836 raft_consensus.cc:348] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.581494 17836 raft_consensus.cc:729] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: af704d4ab2f2496a833c692bd3d25fde, State: Initialized, Role: LEARNER
I20240502 14:17:56.581733 17836 consensus_queue.cc:260] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.582826 17836 ts_tablet_manager.cc:1398] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde: Time spent starting tablet: real 0.002s user 0.000s sys 0.004s
I20240502 14:17:56.583535 12592 tablet_copy_service.cc:342] P 53479f0fb09c454e9b3cbe7398bf8874: Request end of tablet copy session af704d4ab2f2496a833c692bd3d25fde-cd382683ec2b4600b6edaced79698b7e received from {username='slave'} at 127.27.220.132:56877
I20240502 14:17:56.583627 12700 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.583700 12592 tablet_copy_service.cc:433] P 53479f0fb09c454e9b3cbe7398bf8874: ending tablet copy session af704d4ab2f2496a833c692bd3d25fde-cd382683ec2b4600b6edaced79698b7e on tablet cd382683ec2b4600b6edaced79698b7e with peer af704d4ab2f2496a833c692bd3d25fde
I20240502 14:17:56.583907 15014 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.584759 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.585143 14884 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.585269 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.587126 12963 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.587280 12570 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.587877 12306 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.587841 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.588788 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.589988 12831 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.590620 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.592496 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.592569 12440 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.593060 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.601238 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.616428 17759 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.618847 12308 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.625264 12165 catalog_manager.cc:5484] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 reported cstate change: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.629906 14743 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.631734 13094 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.636610 12834 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.641256 15016 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.654780 17846 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.655668 14745 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 14:17:56.656288 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.657709 12961 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.658219 12962 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.669898 14883 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.675145 12161 catalog_manager.cc:5484] T f358e8fda9ca4eeeb1dee5b2011ac029 P d693e089cfa54fbb8df91939410b19db reported cstate change: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.679904 12569 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.681659 14883 raft_consensus.cc:1203] T 00aebb2064324c3eac8145d1c455622c P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 14:17:56.692380 12831 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.694340 16957 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.705291 14884 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.705528 12309 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.710556 12166 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf reported cstate change: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 14:17:56.724017 17553 raft_consensus.cc:1050] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER 95e58ca33b4048e2b000dfbf2b5a0186 to VOTER
I20240502 14:17:56.724472 12831 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.725411 17553 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 14, Committed index: 14, Last appended: 1.14, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:56.728073 15014 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.729317 12963 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.734364 14884 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.737908 17930 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 14:17:56.739961 12440 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.745879 12700 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.746582 17930 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.748236 12702 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.754148 14743 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.760002 14746 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.760685 17930 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.761201 16957 raft_consensus.cc:1013] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: attempt to promote peer af704d4ab2f2496a833c692bd3d25fde: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 14:17:56.761406 12306 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.761945 14746 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.768261 12833 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.769968 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 14:17:56.777467 15016 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.778266 17930 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 14:17:56.782164 17527 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.783002 15016 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.791385 12164 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 reported cstate change: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.802099 13093 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.805028 12700 raft_consensus.cc:1203] T b6bcd8515d234027abfd01ea5dbb3ede P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 14:17:56.846982 12965 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.848017 16957 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 14:17:56.848526 14884 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.850288 12965 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.849442 12700 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.856367 12309 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.857090 17933 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 14:17:56.859359 12309 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.863729 14743 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.871520 12438 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.872622 17945 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.002s
I20240502 14:17:56.874890 16957 raft_consensus.cc:1050] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf: attempting to promote NON_VOTER af704d4ab2f2496a833c692bd3d25fde to VOTER
I20240502 14:17:56.877027 12833 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.880239 16957 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 15, Committed index: 15, Last appended: 1.15, Last appended by leader: 0, Current term: 1, Majority size: 6, State: 0, Mode: LEADER, active raft config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:56.881912 12569 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 14:17:56.883668 12439 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false replace: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.888648 12438 consensus_queue.cc:237] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } }
I20240502 14:17:56.888926 12569 consensus_queue.cc:237] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:56.892429 15016 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 14:17:56.895011 14743 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.895340 12569 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.895609 12831 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.895859 17846 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.896221 17657 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.896536 17351 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.898602 12834 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.898787 15016 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.900666 13095 raft_consensus.cc:1261] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer eddeff583cf34c8dab2942fb36abbf9e: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.901105 16680 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.902554 17524 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.902765 12307 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.903420 16680 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.904193 17846 consensus_queue.cc:1035] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.907123 17351 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.907508 17749 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.908131 12965 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.908011 12307 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 70a425a77fe54cbcb9755f9c540fd367 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.908824 16680 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.909642 17945 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 14:17:56.910082 12831 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.913882 12152 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a6d444ae07234a30be060fbcc5b63b4f with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:56.914499 15015 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.915263 12164 catalog_manager.cc:5484] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 reported cstate change: config changed from index 9 to 10, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New cstate: current_term: 1 leader_uuid: "53479f0fb09c454e9b3cbe7398bf8874" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:56.915290 12834 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P c78bd31d8d2142bf9d8e06b5a0074e0f [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.913147 12965 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.921816 14724 tablet_service.cc:1514] Processing DeleteTablet for tablet a6d444ae07234a30be060fbcc5b63b4f with delete_type TABLET_DATA_TOMBSTONED (TS 4c04a4a8d8864267be7f67c2abe98f84 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:55362
I20240502 14:17:56.927650 14743 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 14:17:56.928411 17945 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 14:17:56.928997 14884 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 14:17:56.929148 12962 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 14:17:56.930078 12441 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 14:17:56.930835 12570 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 14:17:56.931383 12153 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f358e8fda9ca4eeeb1dee5b2011ac029 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 14:17:56.931710 17945 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 14:17:56.932008 17930 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 14:17:56.932380 17553 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 14:17:56.933189 17527 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 14:17:56.934303 12570 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 70a425a77fe54cbcb9755f9c540fd367 (127.27.220.129) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } } }
I20240502 14:17:56.938350 17628 raft_consensus.cc:1050] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db: attempting to promote NON_VOTER 1ba6a238e6ad4e4a84eff746ff336144 to VOTER
I20240502 14:17:56.938433 12572 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.938696 14743 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } }
I20240502 14:17:56.939599 17628 consensus_queue.cc:237] T 9cd04934b5264d7a9643974c0386cac4 P d693e089cfa54fbb8df91939410b19db [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } attrs { replace: true } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false replace: true } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } }
I20240502 14:17:56.940339 12162 catalog_manager.cc:5484] T f358e8fda9ca4eeeb1dee5b2011ac029 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 9 to 10, VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) evicted. New cstate: current_term: 1 leader_uuid: "eddeff583cf34c8dab2942fb36abbf9e" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 14:17:56.950299 12570 raft_consensus.cc:858] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }.
Pending config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }:
New config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:56.950815 12700 raft_consensus.cc:1261] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Refusing update from remote peer 84d1fcd2d4cf472db7959f63a7f0ddaf: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
W20240502 14:17:56.951298 12570 raft_consensus.cc:1565] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Could not prepare op 'term: 1 index: 16' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }.
Pending config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }:
New config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:56.951750 17945 consensus_queue.cc:1035] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Connected to new peer: Peer: permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 14:17:56.953576 12441 raft_consensus.cc:858] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }.
Pending config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }:
New config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
W20240502 14:17:56.953766 12441 raft_consensus.cc:1565] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Could not prepare op 'term: 1 index: 16' and following 0 ops. Status for this op: Illegal state: RaftConfig change currently pending. Only one is allowed at a time.
Committed config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: NON_VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: true } }.
Pending config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }:
New config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:56.953914 12570 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.954346 12441 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, 95e58ca33b4048e2b000dfbf2b5a0186 (127.27.220.137) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: NON_VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: true } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.956825 12438 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.958765 17657 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.958943 13094 raft_consensus.cc:1261] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Refusing update from remote peer 53479f0fb09c454e9b3cbe7398bf8874: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 14:17:56.959609 17657 consensus_queue.cc:1035] T a6d444ae07234a30be060fbcc5b63b4f P 53479f0fb09c454e9b3cbe7398bf8874 [LEADER]: Connected to new peer: Peer: permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 14:17:56.961241 13096 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.965975 17527 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 LEADER]: Committing config change with OpId 1.16: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.969273 12438 raft_consensus.cc:2932] T a6d444ae07234a30be060fbcc5b63b4f P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 4c04a4a8d8864267be7f67c2abe98f84 (127.27.220.136) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.968271 12437 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.971398 14743 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 4c04a4a8d8864267be7f67c2abe98f84 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.976841 14884 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 1ba6a238e6ad4e4a84eff746ff336144 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.983160 12962 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P d693e089cfa54fbb8df91939410b19db [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.985733 12161 catalog_manager.cc:5484] T df48755615874edab65577e92249fc78 P eddeff583cf34c8dab2942fb36abbf9e reported cstate change: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" committed_config { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.987236 12569 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 53479f0fb09c454e9b3cbe7398bf8874 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.993079 12702 raft_consensus.cc:1203] T cd382683ec2b4600b6edaced79698b7e P af704d4ab2f2496a833c692bd3d25fde [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 14:17:57.000488 12702 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P af704d4ab2f2496a833c692bd3d25fde [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:56.984984 15014 raft_consensus.cc:2932] T df48755615874edab65577e92249fc78 P 95e58ca33b4048e2b000dfbf2b5a0186 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, af704d4ab2f2496a833c692bd3d25fde (127.27.220.132) changed from NON_VOTER to VOTER. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } attrs { replace: true } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } } }
I20240502 14:17:57.018132 13095 consensus_queue.cc:237] T df48755615874edab65577e92249fc78 P 84d1fcd2d4cf472db7959f63a7f0ddaf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 16, Committed index: 16, Last appended: 1.16, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 17 OBSOLETE_local: false peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } attrs { replace: true } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "1ba6a238e6ad4e4a84eff746ff336144" member_type: VOTER last_known_addr { host: "127.27.220.138" port: 33073 } attrs { promote: false replace: true } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136" port: 41465 } attrs { promote: false } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } attrs { promote: false } } peers { permanent_uuid: "af704d4ab2f2496a833c692bd3d25fde" member_type: VOTER last_known_addr { host: "127.27.220.132" port: 45069 } attrs { promote: false } } peers { permanent_uuid: "95e58ca33b4048e2b000dfbf2b5a0186" member_type: VOTER last_known_addr { host: "127.27.220.137" port: 40043 } attrs { promote: false } }
I20240502 14:17:57.019714 13094 raft_consensus.cc:2932] T f358e8fda9ca4eeeb1dee5b2011ac029 P 84d1fcd2d4cf472db7959f63a7f0ddaf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 1ba6a238e6ad4e4a84eff746ff336144 (127.27.220.138) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "c78bd31d8d2142bf9d8e06b5a0074e0f" member_type: VOTER last_known_addr { host: "127.27.220.133" port: 45069 } } peers { permanent_uuid: "d693e089cfa54fbb8df91939410b19db" member_type: VOTER last_known_addr { host: "127.27.220.134" port: 36329 } } peers { permanent_uuid: "84d1fcd2d4cf472db7959f63a7f0ddaf" member_type: VOTER last_known_addr { host: "127.27.220.135" port: 44203 } } peers { permanent_uuid: "eddeff583cf34c8dab2942fb36abbf9e" member_type: VOTER last_known_addr { host: "127.27.220.130" port: 34491 } } peers { permanent_uuid: "70a425a77fe54cbcb9755f9c540fd367" member_type: VOTER last_known_addr { host: "127.27.220.129" port: 46613 } } peers { permanent_uuid: "53479f0fb09c454e9b3cbe7398bf8874" member_type: VOTER last_known_addr { host: "127.27.220.131" port: 43727 } } peers { permanent_uuid: "4c04a4a8d8864267be7f67c2abe98f84" member_type: VOTER last_known_addr { host: "127.27.220.136"